CLock

,

.gitignore

@@ -197,3 +197,7 @@ node_modules/prettier/esm/parser-postcss.mjs
 node_modules/prettier/esm/parser-typescript.mjs
 node_modules/prettier/esm/parser-yaml.mjs
 node_modules/prettier/esm/standalone.mjs
+cr_results.json
+not_in_db.json
+rr_results.json
+s_results.json

.trunk/.gitignore

@@ -6,8 +6,4 @@
 plugins
 user_trunk.yaml
 user.yaml
-<<<<<<< HEAD
 tmp
-=======
-tools
->>>>>>> c7dc465 (Parametryzacja "muzyki mojego ludu")

.trunk/configs/.shellcheckrc

@@ -0,0 +1,7 @@
+enable=all
+source-path=SCRIPTDIR
+disable=SC2154
+
+# If you're having issues with shellcheck following source, disable the errors via:
+# disable=SC1090
+# disable=SC1091

.trunk/trunk.yaml

@@ -2,72 +2,39 @@
 # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-<<<<<<< HEAD
-  version: 1.20.1
+  version: 1.22.2
 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
     - id: trunk
-      ref: v1.4.5
+      ref: v1.6.0
       uri: https://github.com/trunk-io/plugins
 # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
-=======
-  version: 1.12.1
-plugins:
-  sources:
-    - id: trunk
-      ref: v0.0.22
-      uri: https://github.com/trunk-io/plugins
-lint:
-  enabled:
-    - actionlint@1.6.25
-    - markdownlint@0.35.0
-    - trivy@0.43.1
-    - trufflehog@3.44.0
-    - pylint
-    - codespell
-    - cspell
-    - git-diff-check
-    - eslint
-    - prettier
-    - semgrep
-    - autopep8
-    - bandit
-    - black
-    - flake8
-    - isort
-    - mypy
-    - pylint
-    - ruff
-    - semgrep
-    - yapf
-    - gitleaks
-
->>>>>>> c7dc465 (Parametryzacja "muzyki mojego ludu")
 runtimes:
   enabled:
+    - go@1.21.0
     - node@18.12.1
     - python@3.10.8
-<<<<<<< HEAD
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   enabled:
-    - actionlint@1.6.27
-    - bandit@1.7.8
-    - black@24.3.0
+    - shellcheck@0.10.0
+    - shfmt@3.6.0
+    - taplo@0.8.1
+    - actionlint@1.7.1
+    - bandit@1.7.9
+    - black@24.4.2
     - codespell
-    - checkov@3.2.48
+    - checkov@3.2.139
     - git-diff-check
     - isort@5.13.2
     - pylint
-    - markdownlint@0.39.0
-    - prettier@3.2.5
-    - ruff@0.3.4
-    - trivy@0.50.1
-    - trufflehog@3.71.0
+    - markdownlint@0.41.0
+    - prettier@3.3.2
+    - ruff@0.4.9
+    - trivy@0.52.2
+    - trufflehog@3.78.1
     - yamllint@1.35.1
-=======
->>>>>>> c7dc465 (Parametryzacja "muzyki mojego ludu")
 actions:
   disabled:
     - trunk-announce

cogs/music.py

@@ -0,0 +1,46 @@
+import discord
+from discord.ext import commands
+from typing import  Optional
+
+class Music(commands.Cog):
+    def __init__(self, bot):
+        self.bot = bot # This is so you can access Bot instance in your cog
+
+# You must have this function for `bot.load_extension` to call
+def setup(bot):
+    bot.add_cog(Music(bot))
+
+@commands.hybrid_command(
+    name="przytul", description="Przytul kogoś - daj mention po komendzie :)"
+)
+async def przytul(ctx, arg: Optional[discord.Member] = None):
+    """
+    Generate a text about hugging mentioned user.
+
+    :param ctx: ctx stands for "context" and is a required parameter in Discord.py commands. It
+    represents the context in which the command was invoked, including information such as the message,
+    the channel, the server, and the user who invoked the command
+    :param arg: arg is a parameter of the function "przytul" that expects a Discord member object. The
+    parameter is optional, meaning that if no member object is provided, it will default to None
+    :type arg: Optional[discord.Member]
+    """
+    async with ctx.typing():
+        nieprzytulac = False
+        for mention in ctx.message.mentions:
+            for role in mention.roles:
+                if role.name == "NIEPRZYTULAĆ!":
+                    nieprzytulac = True
+
+        if arg and nieprzytulac:
+            await ctx.send(
+                f"Żebym ja Ciebie nie przytulił {ctx.message.author.mention}"
+            )
+        elif arg:
+            await ctx.send(
+                # trunk-ignore(codespell/misspelled)
+                f"Już dobrze.... Już dobrze... Ojej.. Biedactwo... :( *W ułamku sekundy {arg.mention} znajduje sie w duszącym uścisku. Żebra trzeszczą - kilka pęka. Pacnięcia po plecach grożą odbiciem nerek, a głaskanie po głowie powoduje wstrząs mózgu*"
+            )
+        else:
+            await ctx.send(
+                "Kogo mam przytulić? *Wyłamuje kostki i przeciąga się - jego 200 kilowa sylwetka złożona z samych mięśni świadczy o tym że jest gotowy*"
+            )

communication_subroutine.py

@@ -0,0 +1,212 @@
+import logging
+import threading
+import json
+import time
+import re
+from urllib import request as urequest
+
+from queue import Queue, Empty
+from flask import Flask, jsonify, request
+from waitress import serve
+
+HOST_ADDRESS = "192.168.1.191"
+PORT_ADDRESS = 5000
+ICECAST_ADDRESS = "http://192.168.1.15:8000"
+OUT_COMM_Q = Queue()
+IN_COMM_Q = Queue()
+SRCHTITLE = re.compile(br'StreamTitle=\\*(?P<title>[^;]*);').search
+
+awaiting_q = []
+incoming_q = Queue()
+app = Flask(__name__)
+PREPPED_TRACKS = {
+    "requests" : "",
+    "hit" : "",
+    "all": "",
+    "priority":"",
+    "jingles":"",
+    "now_playing":"",
+    "next": "",
+    "meta" : ""
+}
+logger = logging.getLogger("discord")
+
+class QueryControl:
+    """
+    This class `QueryControl` is used to manage queries with information about the author, UUID,
+    content, logger, context, and replies.
+    """
+    def __init__(self, query_author, query_uuid, query_content, ctx) -> None:
+        self.author = query_author
+        self.uuid = query_uuid
+        self.content = query_content
+        self.logger = logging.getLogger("discord")
+        self.stop = False
+        self.ctx = ctx
+        self.logger.info(
+            f"Created Query control for {self.author}, {self.uuid}: {self.content}"
+        )
+        self.replies = []
+
+@app.route("/prepped_tracks", methods=["POST"])
+def log_radio_tracks():
+    app.logger = logging.getLogger("discord")
+
+    app.logger.info(request)
+    record = json.loads(request.data)
+    app.logger.info(record)
+    if "next" in record[0]:
+        metadata = id3(ICECAST_ADDRESS)
+        PREPPED_TRACKS["now_playing"] = PREPPED_TRACKS["next"]
+        if metadata:
+            PREPPED_TRACKS["meta"] = metadata["name"] + " - " + metadata["title"] + "(" + metadata["genre"] + ")"
+        else:
+            PREPPED_TRACKS["meta"] = "Nie znaju"
+    PREPPED_TRACKS[record[0]] = record[1]
+    app.logger.info("DATA RECEIVED")
+    return jsonify("SUCCESS")
+
+
+@app.route("/conjurer", methods=["POST"])
+def answer_external_command():
+    """
+    The function `answer_external_command` logs the request data, loads the data as JSON, logs the
+    record, and then puts the record into an incoming queue before returning a success message.
+    :return: The function `answer_external_command()` is returning a JSON response with the message
+    "SUCCESS".
+    """
+    logger = logging.getLogger("discord")
+    logger.info(request)
+    record = json.loads(request.data)
+    logger.info(record)
+    logger.info("DATA RECEIVED")
+    incoming_q.put(record)
+    return jsonify("SUCCESS")
+
+@app.route("/conjurer", methods=["GET"])
+def check_alive():
+    """
+    The function "check_alive" returns a JSON response with the message "ALIVE" when the "/conjurer"
+    route is accessed via a GET request.
+    :return: The code snippet is a Flask route that listens for GET requests to the "/conjurer"
+    endpoint. When a GET request is made to this endpoint, the function check_alive() is called, which
+    returns a JSON response with the message "ALIVE".
+    """
+    return jsonify("ALIVE")
+
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    logger = logging.getLogger("discord")
+
+    logger.info("Attempt debug")
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    logger = logging.getLogger("discord")
+
+    logger.info("Attempt waitress")
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def scan_queue():
+    """
+    The function `scan_queue` reads data from a queue, logs it, and appends it to another queue.
+
+    :param _logger: The `_logger` parameter is typically an instance of a logging object that is used to
+    record and store log messages. It is commonly used to track the flow of the program, record errors,
+    and provide information for debugging purposes. In this code snippet, the `_logger` object is used
+    to log the
+    """
+    logger = logging.getLogger("discord")
+    while True:
+        data = OUT_COMM_Q.get()
+        logger.info(data)
+        awaiting_q.append(data)
+
+def scan_incoming():
+    """
+    The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data
+    is found.
+
+    :param _logger: The `_logger` parameter in the `scan_incoming` function is a logger object that is
+    used to log messages or information during the execution of the function. It is typically used for
+    debugging, monitoring, or tracking the flow of the program. In this case, the `_logger` is being
+    used
+    """
+    logger = logging.getLogger("discord")
+    while True:
+        try:
+            answer = incoming_q.get(block=False)
+            logger.info("DATA FOUND")
+            record_stored = False
+            for record in awaiting_q:
+                if record.uuid in answer.keys():
+                    record_stored = True
+                    record.stop = True
+                    record.entries = answer[record.uuid]
+                    IN_COMM_Q.put(record)
+            if not record_stored:
+                for key in answer.keys():
+                    record = QueryControl("Orphaned", key, "Orphan", None)
+                    record.stop = True
+                    record.entries = answer[record.uuid]
+                    IN_COMM_Q.put(record)
+        except Empty:
+            time.sleep(1)
+
+def get_stream_title(tag:bytes) -> str:
+    title = ''
+    if m := SRCHTITLE(tag):
+        #decode, strip, unescape and remove surrounding quotes (may not even be the same type of quote)
+        title = m.group('title').decode('utf-8').strip().replace('\\', '')[1:-1]
+    return title
+
+def id3(url:str) -> dict:
+    request = urequest.Request(url, headers={'Icy-MetaData': 1})
+
+    with urequest.urlopen(request) as resp:
+        metaint = int(resp.headers.get('icy-metaint', '-1'))
+        if metaint<0:
+            return False
+        resp.read(metaint) #this isn't seekable so, arbitrarily read to the point we want
+        tagdata = dict(
+            site_url = resp.headers.get('icy-url'  )        ,
+            name     = resp.headers.get('icy-name' ).title(),
+            genre    = resp.headers.get('icy-genre').title(),
+            title    = get_stream_title(resp.read(255))     )
+        return tagdata
+
+
+def comm_subroutine():
+    """
+    The `comm_subroutine` function starts multiple threads to run different tasks concurrently.
+
+    :param logger: The `logger` parameter in the `comm_subroutine` function is an instance of a logger
+    object that is used for logging messages at various levels (e.g., debug, info, warning, error). In
+    the provided code snippet, the logger is used to log messages at the "info" level
+    """
+    #logger.setLevel(logging.DEBUG)
+    logger = logging.getLogger("discord")
+    logger.info("Started comms")
+    threads = []
+    #threads.append(threading.Thread(target=flask_debug))
+    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=scan_queue))
+    threads.append(threading.Thread(target=scan_incoming))
+
+
+    for worker in threads:
+        worker.start()
+    for worker in threads:
+        worker.join()
+
+if __name__ == "__main__":
+    comm_subroutine()

conjurer.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/bot.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

conjurer_librarian/conjurer_librarian.py

@@ -0,0 +1,507 @@
+"""
+This module contains the implementation of the Librarian class and
+related functions for searching and refining queries.
+
+Classes:
+-   Librarian: Represents a librarian object that performs search and
+    refinement operations on queries
+
+Functions:
+-   flask_debug: Starts a Flask application in debug mode without using the reloader.
+-   waitress_run: Serves the Flask application using the Waitress WSGI server.
+-   BackgroundTaskSearch: Represents a background task for running the
+    Librarian object asynchronously
+"""
+
+import asyncio
+import json
+import logging
+import netrc
+import threading
+import time
+from json.decoder import JSONDecodeError
+from logging import handlers
+from queue import Queue
+
+import requests
+import scrape_bot
+import search_bot
+#import search_bot2 as search_bot
+from flask import Flask, jsonify, request
+from habanero import Crossref
+from waitress import serve
+
+# Constants
+NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
+HOST_ADDRESS = "192.168.1.192"
+PORT_ADDRESS = 5001
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+SEND_RESULTS = "/conjurer"
+BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
+
+MAX_CR_RESULTS = 1000
+#TEST PURPOSES ONLY!
+#MAX_CR_RESULTS = 5
+
+ENCODING = "utf-8"
+
+app = Flask(__name__)
+
+librarian_queue = Queue()
+librarian_list = []
+
+
+# trunk-ignore(pylint/R0902)
+class Librarian(object):
+    """
+    Represents a librarian object that performs search and refinement operations on queries.
+    """
+
+    def __init__(self, _app, query, uuid, _deep_search) -> None:
+        """
+        Initializes a Librarian object.
+
+        Args:
+        - _app: The Flask application object.
+        - query: The query to be searched.
+        - uuid: The unique identifier for the search.
+
+        Attributes:
+        - cr: The Crossref object for performing the search.
+        - query: The query to be searched.
+        - uuid: The unique identifier for the search.
+        - limit: The maximum number of search results to fetch.
+        - fetched: The number of search results fetched so far.
+        - hit: The number of search results that match the refinement criteria.
+        - total: The total number of search results.
+        - app: The Flask application object.
+        - live_results: A list to store live search results.
+        - final_result: A list to store the final refined search results.
+        - not_in_db: A list to store search results that are not in the local database.
+        - search_result_from_cr: A dictionary to store the search results from Crossref.
+        - done: A flag indicating if the search is done.
+        """
+        netrc_mod = netrc.netrc(NETRC_FILE)
+        auth_tokens = netrc_mod.authenticators("crossref")
+        self.cr = Crossref(
+            mailto=auth_tokens[0],
+            ua_string=f"Conjurer project. mailto:{auth_tokens[0]}"
+        )
+        self.query = query
+        self.uuid = str(uuid)
+        self.limit = MAX_CR_RESULTS
+        self.fetched = 0
+        self.hit = 0
+        self.total = 0
+        self.app = _app
+        self.live_results = []
+        self.final_result = {}
+        self.not_in_db = {}
+        self.search_result_from_cr = {}
+        self.done = False
+        self.deep_search = _deep_search
+
+    async def search_crossref(self, query, deep_search=False):
+        """
+        Performs a search on Crossref for the given query.
+
+        Args:
+        - query: The query to be searched.
+
+        Returns:
+        - result: The search result from Crossref.
+
+        Raises:
+        - None.
+        """
+
+        self.app.logger.info("STARTED SEARCH")
+        
+        if not deep_search:
+            query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000
+            cr_result = self.cr.works(query=query, limit=query_limit)
+            self.search_result_from_cr.update(cr_result)
+            self.total = cr_result["message"]["total-results"]
+            self.fetched += len(cr_result["message"]["items"])
+            self.app.logger.info(self.total)
+            self.app.logger.info(self.fetched)
+            while self.total > self.fetched and self.limit > self.fetched:
+                tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched)
+                cr_result["message"]["items"].extend(tmp_result["message"]["items"])
+                self.total = tmp_result["message"]["total-results"]
+                self.fetched = len(cr_result["message"]["items"])
+                self.app.logger.info(self.total)
+                self.app.logger.info(self.fetched)
+                time.sleep(0.1)
+
+        else:
+            cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
+            result = cr_result[0]
+            for item in cr_result[1:]:
+                result["message"]["items"].extend(item["message"]["items"])
+                self.total = item["message"]["total-results"]
+                self.fetched = len(result["message"]["items"])
+                self.app.logger.info(self.total)
+                self.app.logger.info(self.fetched)
+            cr_result = result
+            self.app.logger.info("Total, fetched:")
+            self.app.logger.info(self.total)
+            self.app.logger.info(self.fetched)
+            self.search_result_from_cr.update(cr_result)
+            self.total = cr_result["message"]["total-results"]
+            self.app.logger.info("CROSSREF DONE")
+
+            self.app.logger.info("CROSSREF DONE")
+        with open("cr_results.json", "r+", encoding="utf-8") as data_file:
+            # First we load existing data into a dict.
+            try:
+                file_data = json.load(data_file)
+            except JSONDecodeError:
+                file_data = {}
+            data_file.truncate(0)
+            data_file.seek(0)
+            tmp = {self.uuid : self.search_result_from_cr}
+            if file_data:
+                file_data.update(tmp)
+            else:
+                file_data = tmp
+            json.dump(file_data, data_file, indent=4)
+        return cr_result
+
+
+    async def refine_search(self, unrefined_result):
+        """
+        Refines the search query based on the unrefined search result.
+
+        Args:
+        - unrefined_result: The unrefined search result.
+
+        Returns:
+        - refined_result: The refined search result.
+
+        Raises:
+        - None.
+        """
+        summarized_results = []
+        self.app.logger.info("REFINE: Removing all derived works from the list")
+        for item in unrefined_result["message"]["items"]:
+            summarized_results.append(
+                {
+                    "DOI": item["DOI"],
+                    "title": item["title"] if "title" in item else None,
+                    "type": item["type"] if "type" in item else None,
+                }
+            )
+
+        partial_result = {
+            self.uuid: {
+                "total_results": self.search_result_from_cr["message"][
+                    "total-results"
+                ],
+                "on_page": 1,
+                "summary": summarized_results,
+                "results": self.search_result_from_cr["message"]["items"],
+            }
+        }
+        self.app.logger.info("REFINE: Dumping to file")
+        temp = []
+        refined_result = {}
+        for key in partial_result:
+            self.app.logger.info("KEY:")
+            self.app.logger.info(key)
+
+        for item in partial_result[self.uuid]["summary"]:
+            if item["title"]:
+                temp.append(item)
+
+        for item in temp:
+            refined_result[item["DOI"]]=  item
+        with open("rr_results.json", "r+", encoding="utf-8") as data_file:
+            # First we load existing data into a dict.
+            try:
+                file_data = json.load(data_file)
+            except JSONDecodeError:
+                file_data = {}
+            data_file.truncate(0)
+            data_file.seek(0)
+            tmp = {self.uuid: refined_result}
+            if file_data:
+                file_data.update(tmp)
+            else:
+                file_data = tmp
+            json.dump(file_data, data_file, indent=4)
+        return refined_result
+
+    async def check_if_exists(self, refined_result):
+        """
+        Checks if the given DOI exists.
+
+        Args:
+        - doi: The DOI to be checked.
+        - brute_force: A flag indicating if brute force method should be used.
+
+        Returns:
+        - result: The search result.
+
+        Raises:
+        - None.
+        """
+        result = {}
+        self.app.logger.info("REFINE: Running search in the backend app")
+        dois = []
+        for item, value in refined_result.items():
+            dois.append([item, value])
+        coro = asyncio.to_thread(
+            search_bot.search_for_doi, dois, self.live_results, self.app.logger
+        )
+        result = await coro
+        result_list = []
+        result_no_db = []
+        for item in result:
+            if item["exists"]:
+                result_list.append(item)
+            else:
+                result_no_db.append(item)
+        self.hit = len(result)
+        return result_list, result_no_db
+
+
+    async def answer_query(self, deep_search=False):
+        """
+        Answers the search query.
+
+        Args:
+        - None.
+
+        Returns:
+        - result: The search result.
+
+        Raises:
+        - None.
+        """
+        self.app.logger.info(f"Search started {self.uuid}")
+        cr_result = await self.search_crossref(query=self.query, deep_search=deep_search)
+        refined_result = await self.refine_search(cr_result)
+        answer, negative_answer = await self.check_if_exists(refined_result)
+
+        self.app.logger.info("Returning result")
+        self.app.logger.info(answer)
+        self.app.logger.info(negative_answer)
+
+        for item in answer:
+            self.final_result[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
+        for item in negative_answer:
+            self.not_in_db[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
+        self.app.logger.info("Returning result case2")
+        self.app.logger.info(self.final_result)
+        return self.final_result
+
+# ============================= FLASK INTERNALS===============================
+
+
+def flask_debug():
+    """
+    Starts a Flask application in debug mode without using the reloader.
+
+    Args:
+    - None.
+
+    Returns:
+    - None.
+
+    Raises:
+    - None.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+def waitress_run():
+    """
+    Serves the Flask application using the Waitress WSGI server.
+
+    Args:
+    - None.
+
+    Returns:
+    - None.
+
+    Raises:
+    - None.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+class BackgroundTaskSearch(threading.Thread):
+    """
+    A background task for searching and saving results to files.
+
+    This class extends the `threading.Thread` class and is responsible for running
+    the search task in the background. It retrieves queries from a queue, performs
+    the search, and saves the results to files.
+
+    Attributes:
+        app (App): The application instance.
+    """
+
+    def run(self):
+        """
+        Run the background task.
+
+        This method is called when the thread is started. It creates a new event loop,
+        runs the `_run` method, and closes the event loop.
+        """
+        loop = asyncio.new_event_loop()
+        loop.run_until_complete(self._run())
+        loop.close()
+
+    async def _run(self):
+        """
+        Perform the search task.
+
+        This method is an asynchronous coroutine that runs in a loop. It retrieves a
+        librarian from the queue, answers the query, and saves the results to files.
+        It also sends the results to a remote server.
+
+        The search task continues running indefinitely until the thread is stopped.
+        """
+        while True:
+            database = None
+            ndb_database = None
+            librarian = librarian_queue.get()
+            self.app.logger.info("STARTED")
+            result = await librarian.answer_query(librarian.deep_search)
+            result = {librarian.uuid: result}
+            self.app.logger.info("Saving to file")
+
+            # Save results to "not_in_db.json" file
+            with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
+                try:
+                    ndb_database = json.load(ndb_file)
+                except JSONDecodeError:
+                    pass
+                if ndb_database:
+                    ndb_database.update(librarian.not_in_db)
+                else:
+                    ndb_database = librarian.not_in_db
+                ndb_file.truncate(0)
+                ndb_file.seek(0)
+                json.dump(ndb_database, ndb_file)
+
+            # Save results to "s_results.json" file
+            with open("s_results.json", "r+", encoding="utf-8") as s_file:
+                try:
+                    database = json.load(s_file)
+                except JSONDecodeError:
+                    pass
+                if database:
+                    self.app.logger.info(database)
+                    self.app.logger.info(result)
+                    database.update(result)
+                else:
+                    database = result
+                self.app.logger.info("DUMPING DATA")
+                s_file.truncate(0)
+                s_file.seek(0)
+                json.dump(database, s_file)
+            self.app.logger.info("FINISHED")
+
+            self.app.logger.info(result)
+            coroutine = asyncio.to_thread(
+                requests.post,
+                f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+                json=result,
+                timeout=360,
+            )
+            self.app.logger.info("SENT")
+            result = await coroutine
+            self.app.logger.info(result.status_code)
+            self.app.logger.info("SEND CONFIRMED")
+            time.sleep(1)
+
+
+# ==================================SERVER ROUTES==========================================
+@app.route("/query", methods=["POST"])
+async def query_database():
+    """
+    Endpoint for querying the database.
+
+    This function receives a POST request containing a JSON payload with a query and a UUID.
+    It creates a Librarian object with the query and UUID,
+    and adds it to the librarian_queue and librarian_list.
+    Finally, it returns a JSON response indicating the success
+    of the operation, along with the query, UUID,
+    and the current size of the librarian_queue.
+
+    Returns:
+        tuple: A tuple containing a JSON response and a status code.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["query"])
+    app.logger.info(record["UUID"])
+    uuid = record["UUID"]
+    deep_search = record["deep_search"]
+    cl = Librarian(app, record["query"], uuid, deep_search)
+    librarian_queue.put(cl)
+    librarian_list.append(cl)
+    answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
+        200,
+    )
+    return return_data
+
+
+@app.route("/get_partial_result", methods=["POST"])
+async def get_partial():
+    """
+    Retrieves the partial result for a given UUID.
+
+    Returns:
+        A JSON response containing the partial result.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["UUID"])
+    for lib in librarian_list:
+        if lib.uuid == record["UUID"]:
+            answer_data = lib.live_results
+            break
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
+        200,
+    )
+    return return_data
+
+
+# =======================================MAIN===================================================
+if __name__ == "__main__":
+    app.logger.setLevel(logging.DEBUG)
+    h1 = handlers.RotatingFileHandler(
+        filename="D:\\logs\\librarian.log",
+        encoding="utf-8",
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
+    )
+
+    app.logger.addHandler(h1)
+    threads = []
+    threads.append(threading.Thread(target=waitress_run))
+    # threads.append(threading.Thread(target=flask_debug))
+    bgtask = BackgroundTaskSearch()
+    bgtask.app = app
+    threads.append(bgtask)
+    threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
+    i = 0
+    for worker in threads:
+        try:
+            app.logger.info("App number: %s", i)
+            i += 1
+            worker.start()
+        except RuntimeError as e:
+            app.logger.error("Exploded")
+            print(str(e))
+    for worker in threads:
+        worker.join()

conjurer_librarian/count_lines.py

@@ -0,0 +1,8 @@
+
+FILEPTH = r"C:\Database\chunks\4_chunk.txt"
+
+with open(FILEPTH, "r", encoding="utf-8") as file:
+    tab = file.readlines()
+    print(len(tab))
+
+    #1842295

conjurer_librarian/install_librarian_service.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# This script sets up a virtual environment for the Conjurer Librarian service and installs the required dependencies.
+
+
+# Create a virtual environment at /home/pi/Conjurer_librarian/env
+python3 -m venv /home/pi/Conjurer_librarian/env
+
+# Copy the Conjurer Librarian Python script to the appropriate directory
+cp /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+
+# Copy the requirements file to the appropriate directory
+cp /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
+
+# Change directory to /home/pi/Conjurer_librarian
+cd /home/pi/Conjurer_librarian
+
+touch /home/pi/Conjurer/rr_results.json
+touch /home/pi/Conjurer/cr_results.json
+touch /home/pi/Conjurer/s_results.json
+touch /home/pi/Conjurer/rr_results.json
+touch /home/pi/Conjurer/not_in_db.json
+
+# Activate the virtual environment
+source /home/pi/Conjurer_librarian/env/bin/activate
+
+# Upgrade pip
+./env/bin/python3 -m pip install --upgrade pip
+
+# Install the required dependencies
+./env/bin/python3 -m pip install -r requirements_librarian.txt
+
+# Deactivate the virtual environment
+deactivate

conjurer_librarian/requirements_librarian.txt

@@ -0,0 +1,3 @@
+habanero
+waitress
+flask[async]

conjurer_librarian/run_server.bat

@@ -0,0 +1 @@
+c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py

conjurer_librarian/scrape_bot.py

@@ -0,0 +1,136 @@
+"""
+This module contains the code for the scrape bot.
+"""
+
+import json
+import logging
+import random
+import re
+import time
+from json import JSONDecodeError
+from queue import Queue
+from threading import Thread
+from urllib.request import urlopen
+
+from requests import ConnectionError as RequestsConnectionError
+from requests import ConnectTimeout, Timeout
+
+SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
+SCR_FILENAME = "40_chunk.txt"
+SCR_ENCODING = "utf-8"
+
+WORK_Q = Queue()
+random.seed()
+
+
+def load_ndb_to_q(logger):
+    """
+    This function loads items from the not_in_db.json file into the work queue.
+    """
+    logger.info("Loader started")
+    while True:
+        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
+            try:
+                ndb_database = json.load(ndb_file)
+                for _ in range (1,10):
+                    try:
+                        key = next(iter(ndb_database))
+                        _ = ndb_database.pop(key)
+                        logger.info(key)
+                        url = f"https://sci-hub.se/{key}"
+                        WORK_Q.put([key, url, False])
+                    except StopIteration:
+                        break
+                ndb_file.truncate(0)
+                ndb_file.seek(0)
+                json.dump(ndb_database, ndb_file, indent=4)
+            except JSONDecodeError:
+                time.sleep(60 * 60 * 3)
+        time.sleep(60*60*3)
+
+
+def check_if_exists_brute_force(logger):
+    """
+    This function checks if a document exists using brute force search.
+    """
+    # Function code here
+    while True:
+        logger.info("Scraper tick")
+        item = WORK_Q.get()
+        page_url = item[1]
+        logger.error("REFINE: Brute force search!")
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        blocked = True
+        try:
+            # trunk-ignore(bandit/B310)
+            with urlopen(page_url) as response:
+                data = response.read()
+                text = data.decode("utf-8")
+                if item[2]:
+                    logger.info("Already found")
+                    time.sleep(180)
+                    blocked = False
+                else:
+                    for line in text.splitlines():
+                        if re.match(
+                            r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
+                            line,
+                        ):
+                            blocked = False
+                            logger.info("Not found")
+                            item[2] = False
+                        if m := re.match(
+                            r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
+                        ):
+                            blocked = False
+                            direct_download_link = "https:" + str(m.group(1))
+                            logger.info(direct_download_link)
+                            with open(
+                                SCR_DATABASE_PATH + SCR_FILENAME,
+                                "a",
+                                encoding=SCR_ENCODING,
+                            ) as operated_file:
+                                operated_file.write("\n")
+                                operated_file.write(item[0])
+                                item[2] = True
+        except (
+            Timeout,
+            ConnectTimeout,
+            ConnectionRefusedError,
+            ConnectionError,
+            RequestsConnectionError,
+        ):
+            pass
+        if blocked:
+            logger.info(item)
+            logger.info(text)
+            logger.error("Got blocked. Fuck.")
+            time.sleep(60 * 60 * 72)
+        # trunk-ignore(bandit/B311)
+        rand = random.randint(1, 60)
+        logger.info(f"Sleeping for {2*rand} minutes")
+        time.sleep(120 * rand)
+
+
+def scraper(logger=None):
+    """
+    This function is responsible for scraping data.
+    """
+    if not logger:
+        logger = logging.getLogger()
+        logger.setLevel("DEBUG")
+        h1 = logging.StreamHandler()
+        logger.addHandler(h1)
+        logger.info("No logger set. Test run")
+
+    loader = Thread(target=load_ndb_to_q, args=(logger,))
+    _scraper = Thread(target=check_if_exists_brute_force, args=(logger,))
+    loader.start()
+    _scraper.start()
+    loader.join()
+    _scraper.join()
+
+
+if __name__ == "__main__":
+    scraper()

conjurer_librarian/search_bot.py

@@ -0,0 +1,200 @@
+"""
+This module contains functions for searching for DOI (Digital Object Identifier)
+in a list of live results.
+It includes a producer-consumer pattern implementation using threads and queues.
+
+Functions:
+- producer: Reads lines from a file and puts them into an output queue.
+- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
+- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
+
+Global Variables:
+- MAXTHREADS: Maximum number of worker threads.
+- DATABASE_PATH: Path to the database files.
+- ENCODING: Encoding of the database files.
+- CHUNK: File name pattern for the database files.
+- _sentinel: Sentinel object used to signal termination.
+- result_list: List to store the search results.
+- WORK_Q_SIZE: Maximum size of the work queue.
+"""
+
+# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+from queue import Empty, Queue
+from threading import Thread
+import time
+q = Queue()
+#TODO: Count number of lines in files and print to approximate on which part of the file search is
+
+# DATA FOR TEST ONLY
+# MAXTHREADS = 5
+# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+
+# DEPLOYMENT DATA
+MAXTHREADS = 41
+DATABASE_PATH = r"C:\\Database\\chunks\\"
+
+ENCODING = "utf-8"
+CHUNK = "_chunk.txt"
+_sentinel = object()
+WORK_Q_SIZE = 35500000
+
+
+def producer(out_q, control_q, filename, _logger):
+    """
+    Produces items from the output queue and puts them into the control queue.
+
+    Args:
+        out_q (Queue): Output queue.
+        control_q (Queue): Control queue.
+        filename (str): Name of the file.
+        _logger: Logger object for logging.
+    """
+    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+        print(f"Worker {filename} ")
+        line_no = 0
+        while True:
+            line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t \t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+
+            if not line:
+                break
+            out_q.put(line)
+            try:
+                check = control_q.get(block=False)
+            except Empty:
+                check = False
+
+            if check is _sentinel:
+                print("TERM signal received")
+                control_q.put(check)
+                break
+        print(f"Worker finished: {filename}")
+        out_q.put(_sentinel)
+
+
+# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
+    """
+    Consumes items from an input queue and checks if DOI exists in the live results list.
+
+    Args:
+        in_q (Queue): Input queue.
+        control_q (Queue): Control queue.
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    print(f"Consumer thread started: {no} no")
+    empty_counter = 0
+    alive_no = 0
+    while True:
+        done_check = True
+        try:
+            data = in_q.get(block=True, timeout = 1)
+            if data is _sentinel:
+                control_dict["sentinels"] += 1
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
+
+            else:
+                empty_counter = 0
+                alive_no += 1
+                print(f"C{no}__{alive_no}\r", end="")
+
+                for item in result_list:
+                    if item["DOI"] in data and not item["exists"]:
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
+                        _logger.info(data)
+                        _logger.info("HIT")
+                        item["exists"] = True
+                        live_results.append(item)
+                    done_check = done_check and item["exists"]
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+
+        if control_dict["sentinels"] >= MAXTHREADS:
+            _logger.info(f"All workers finished {no}")
+            break
+
+
+
+def search_for_doi(doi, live_results, _logger):
+    """
+    Search for DOI in live_results using _logger for logging.
+
+    Args:
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    control_dict = {"sentinels":0}
+    result_list = []
+    threads = []
+    work_q = Queue(maxsize=WORK_Q_SIZE)
+    control_q = Queue()
+
+    for item in doi:
+        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+
+    for i in range (0, (len(doi)//1000)+2):
+        t_cons = Thread(
+            target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)
+        )
+        _logger.info("Consumer thread created")
+        threads.append(t_cons)
+    for i in range(0, MAXTHREADS):
+        # TEST DATA
+        # filename = "test" + str(i) + CHUNk
+        # DEPLOYMENT DATA
+        filename = str(i) + CHUNK
+        _logger.info(f"Creating worker thread no: {i}")
+        threads.append(
+            Thread(target=producer, args=(work_q, control_q, filename, _logger))
+        )
+    for worker in threads:
+        worker.start()
+    for worker in threads:
+        worker.join()
+    return result_list
+
+
+if __name__ == "__main__":
+    import logging
+
+    logger = logging.getLogger()
+    logger.setLevel("DEBUG")
+    h1 = logging.StreamHandler()
+    logger.addHandler(h1)
+    logger.info("TEST RUN")
+    live_result = []
+    logger.info(
+        search_for_doi(
+            [
+                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
+                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
+                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
+                ("10.2165/00128415-200309690-00017", "DATA"),
+                ("10.5772/48313", "DATA"),
+                ("10.15803/ijnc.7.2_419", "DATA"),
+                ("10.1051/0004-6361/201321596e", "DATA"),
+                ("10.2307/40835941", "DATA"),
+            ],
+            live_result,
+            logger,
+        )
+    )
+    logger.info(live_result)

conjurer_librarian/search_bot2.py

@@ -0,0 +1,206 @@
+"""
+This module contains functions for searching for DOI (Digital Object Identifier)
+in a list of live results.
+It includes a producer-consumer pattern implementation using threads and queues.
+
+Functions:
+- producer: Reads lines from a file and puts them into an output queue.
+- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
+- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
+
+Global Variables:
+- MAXTHREADS: Maximum number of worker threads.
+- DATABASE_PATH: Path to the database files.
+- ENCODING: Encoding of the database files.
+- CHUNK: File name pattern for the database files.
+- _sentinel: Sentinel object used to signal termination.
+- result_list: List to store the search results.
+- WORK_Q_SIZE: Maximum size of the work queue.
+"""
+
+# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+from multiprocessing import Manager, Process, Queue
+from queue import Empty, Full
+import time
+
+# DATA FOR TEST ONLY
+#MAXTHREADS = 6
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+
+# DEPLOYMENT DATA
+MAXTHREADS = 41
+DATABASE_PATH = r"C:\\Database\\chunks\\"
+
+ENCODING = "utf-8"
+CHUNK = "_chunk.txt"
+_sentinel = None
+WORK_Q_SIZE = 355000
+
+
+def producer(out_q, control_q, filename, _logger):
+    """
+    Produces items from the output queue and puts them into the control queue.
+
+    Args:
+        out_q (Queue): Output queue.
+        control_q (Queue): Control queue.
+        filename (str): Name of the file.
+        _logger: Logger object for logging.
+    """
+    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+        print(f"Worker {filename} ")
+        line_no = 0
+        while True:
+            line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+            try:
+                out_q.put([line, line_no, filename], block = True, timeout = 60)
+            except Full:
+                print("queue fulll")
+            try:
+                check = control_q.get(block=False)
+            except Empty:
+                check = False
+            if check is None:
+                print(f"TERM signal received {filename}")
+                control_q.put(check)
+                control_q.close()
+                break
+        print(f"Worker finished: {filename}")
+        out_q.put(_sentinel)
+        out_q.close()
+
+
+
+# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
+    """
+    Consumes items from an input queue and checks if DOI exists in the live results list.
+
+    Args:
+        in_q (Queue): Input queue.
+        control_q (Queue): Control queue.
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    print(f"Consumer thread started: {no} no")
+    empty_counter = 0
+    alive_no = 0
+    while True:
+        done_check = True
+        try:
+            data = in_q.get(block=True, timeout = 1)
+            empty_counter = 0
+            if data is None:
+                control_dict["sentinels"] += 1
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
+            else:
+                empty_counter = 0
+                if alive_no > 9:
+                    alive_no = 0
+                alive_no += 1
+                print(f"C{no}{alive_no}\r", end="")
+                for item in result_list:
+                    if item["DOI"] in data[0] and not item["exists"]:
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
+                        item["exists"] = True
+                        live_results.append(item)
+                    done_check = done_check and item["exists"]
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+        if control_dict["sentinels"] >= MAXTHREADS:
+            print(f"All workers finished killing {no}")
+            break
+    print("Consumer thread finished %s",no )
+    control_q.close()
+
+
+def search_for_doi(doi, live_results, _logger):
+    """
+    Search for DOI in live_results using _logger for logging.
+
+    Args:
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    with Manager() as manager:
+        control_dict = manager.dict()
+        control_dict["sentinels"] = 0
+        result_list = []
+        for item in doi:
+            result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+        result_list_proxy = manager.list(result_list)
+
+        threads = []
+        work_q = Queue(maxsize=WORK_Q_SIZE)
+        control_q = Queue()
+        for i in range(0, MAXTHREADS):
+            # TEST DATA
+            #filename = "test" + str(i) + CHUNK
+            # DEPLOYMENT DATA
+            filename = str(i) + CHUNK
+            print(f"Creating worker thread no: {i}")
+            threads.append(
+                Process(target=producer, args=(work_q, control_q, filename, _logger))
+            )
+        for i in range (0, (len(doi)//1000)+5):
+            t_cons = Process(
+                target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger)
+            )
+            print("Consumer thread created")
+            threads.append(t_cons)
+
+        for worker in threads:
+            worker.start()
+        for worker in threads:
+            worker.join()
+        result_list = list(result_list_proxy)
+        for worker in threads:
+            worker.close()
+        return result_list
+
+
+if __name__ == "__main__":
+    import logging
+
+    logger = logging.getLogger()
+    logger.setLevel("DEBUG")
+    h1 = logging.StreamHandler()
+    logger.addHandler(h1)
+    print("TEST RUN")
+    live_result = []
+    print(
+        search_for_doi(
+            [
+                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
+                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
+                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
+                ("10.2165/00128415-200309690-00017", "DATA"),
+                ("10.5772/48313", "DATA"),
+                ("10.15803/ijnc.7.2_419", "DATA"),
+                ("10.1051/0004-6361/201321596e", "DATA"),
+                ("10.2307/40835941", "DATA"),
+                ("NOT_ON_LIST", "DATA"),
+            ],
+            live_result,
+            logger,
+        )
+    )
+    print(live_result)

conjurer_librarian/test_send.py

@@ -0,0 +1,45 @@
+import asyncio
+import json
+import logging
+from json.decoder import JSONDecodeError
+from logging import handlers
+import requests
+
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+SEND_RESULTS = "/conjurer"
+
+async def send_results():
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    h1 = handlers.RotatingFileHandler(
+        filename="D:\\logs\\librarian.log",
+        encoding="utf-8",
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
+    )
+
+    logger.addHandler(h1)
+
+    with open("s_results.json", "r+", encoding="utf-8") as s_file:
+        try:
+            database = json.load(s_file)
+        except JSONDecodeError:
+            pass
+    result = database["4bacbca0-ce3b-45c6-90a1-05c70b46b740"]
+    result = {"4bacbca0-ce3b-45c6-90a1-05c70b46b740":result}
+
+    coroutine = asyncio.to_thread(
+        requests.post,
+        f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+        json=result,
+        timeout=360,
+    )
+    logger.info("SENT")
+    result = await coroutine
+    logger.info(result.status_code)
+    logger.info("SEND CONFIRMED")
+async def main():
+    await send_results()
+if __name__ == "__main__":
+    asyncio.run(main())

conjurer_musician/conjurer_musician.py

@@ -0,0 +1,580 @@
+# This Python file uses the following encoding: utf-8
+"""
+The provided Python script sets up a Flask web server to manage a list of music files, with
+functions for rescanning the music folder, updating the music list, and serving the music list via
+API endpoints.
+"""
+import json
+import logging
+import os
+import re
+import requests
+import threading
+import time
+from datetime import datetime
+from logging import handlers
+from pathlib import Path
+from platform import uname
+from sys import platform
+from flask import Flask, jsonify, redirect, request, send_from_directory, render_template
+from waitress import serve
+
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+MUSIC_TRACKER = "/prepped_tracks"
+HOST_ADDRESS = "192.168.1.15"
+PORT_ADDRESS = 5000
+if platform in ("linux", "linux2"):
+    SEPARATOR_FILE_PATH = "/"
+    if "microsoft-standard" in uname().release:
+        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
+        MUSIC_FOLDER = "/mnt/g/Muzyka/"
+        NETRC_FILE = "/home/mtuszowski/.netrc"
+        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
+        ENCODING = "utf-8"
+        RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
+        PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
+
+    else:
+        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
+        NETRC_FILE = "/home/pi/.netrc"
+        LOGSTORE = "/home/pi/RetroPie/logs/"
+        ENCODING = "utf-8"
+        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
+        PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/"
+        RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
+        PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
+
+
+music_file_list = []
+priority_list = []
+
+def rescan():
+    """
+    The `rescan` function logs a message, scans for mp3 files in a specified folder,
+    and adds them to a list of music files.
+    """
+    logger = logging.getLogger("conjurer_musician")
+    logger.info("Rescan triggered")
+
+    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+        temp_music_file = mp3_item.as_posix()
+        if platform == "win32":
+            temp_music_file = temp_music_file.replace("/", "\\")
+        music_file_list.append(temp_music_file)
+
+    for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"):
+        temp_music_file = mp3_item.as_posix()
+        if platform == "win32":
+            temp_music_file = temp_music_file.replace("/", "\\")
+        priority_list.append(temp_music_file)
+
+    with open(
+            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
+        ) as w_file:
+        try:
+            for item in music_file_list:
+                w_file.write(item)
+                w_file.write("\n")
+        except json.JSONDecodeError:
+            pass
+    with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
+        try:
+            for item in priority_list:
+                w_file.write(item)
+                w_file.write("\n")
+        except json.JSONDecodeError:
+            pass
+
+
+def thread_rescan():
+    """
+    The `thread_rescan` function periodically triggers a rescan operation after a specified time
+    interval.
+    """
+    logger = logging.getLogger("conjurer_musician")
+    logger.info("Starting filesystemupdater")
+    while True:
+        time.sleep(60 * 60 * 24)
+        logger.info("Rescan triggered")
+        rescan()
+
+
+def scan_tracks():
+    #Set the filename and open the file
+    logger = logging.getLogger("conjurer_musician")
+
+    file = open(RADIOLOG_PATH,'r')
+    #Find the size of the file and move to the end
+    st_results = os.stat(RADIOLOG_PATH)
+    st_size = st_results[6]
+    file.seek(st_size)
+    st_results1 = os.stat(PERSISTENCE_PATH)
+    prev_st_size1 = st_results[6]
+
+    while 1:
+
+        st_results1 = os.stat(PERSISTENCE_PATH)
+        st_size1 = st_results1[6]
+        if prev_st_size1 != st_size1:
+            while prev_st_size1 != st_size1:
+                prev_st_size1 = st_size1
+                st_results1 = os.stat(PERSISTENCE_PATH)
+                st_size1 = st_results1[6]
+                time.sleep(0.1)
+            file1 = open(PERSISTENCE_PATH, 'r')
+            lines = file1.readlines()
+            result = ["next", lines[2]]
+            file1.close()
+            returned = requests.post(
+                    f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
+                    json=result,
+                    timeout=360)
+            logger.info("SENT")
+            logger.info(returned.status_code)
+            logger.info("SEND CONFIRMED")
+
+        where = file.tell()
+        line = file.readline()
+        if not line:
+            time.sleep(1)
+            file.seek(where)
+        else:
+            if re.match(".*Prepared.*",line):
+                result = None
+                if re.match(".*jingles.*", line):
+                    logger.info("jingles")
+                    logger.info(line) # already has newline
+                    result = ["jingles", line]
+                elif re.match(".*priority.*", line):
+                    logger.info("priority")
+                    logger.info(line) # already has newline
+                    result = ["priority", line]
+                elif re.match(".*hit.*", line):
+                    logger.info("hit")
+                    logger.info(line) # already has newline
+                    result = ["hit", line]
+                elif re.match(".*all_playlist.*", line):
+                    logger.info("all")
+                    logger.info(line) # already has newline
+                    result = ["all", line]
+                elif re.match(".*request.*", line):
+                    logger.info("requests")
+                    logger.info(line) # already has newline
+                    result = ["requests", line]
+                if result:
+                    returned = requests.post(
+                    f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
+                    json=result,
+                    timeout=360)
+                    logger.info("SENT")
+                    logger.info(returned.status_code)
+                    logger.info("SEND CONFIRMED")
+        time.sleep(0.1)
+
+
+app = Flask(__name__)
+
+
+# TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
+def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
+    """
+    Searches for music files based on a list of keywords and returns a list of matching files.
+
+    Args:
+        word_list (list): A list of keywords to search for.
+        how_many (int): The maximum number of matching files to return.
+
+    Returns:
+        list: A list of matching files, sorted by their search weight.
+
+    """
+    if _logger is None:
+        fun_logger = logging.getLogger()
+    else:
+        fun_logger = _logger
+    return_list = []
+    search_weight = []
+
+    for _ in range(len(music_file_list)):
+        search_weight.append((0, ""))
+
+    time_start = datetime.now()
+    if int(how_many) > 0:
+        skip_start = 2
+    else:
+        skip_start = 1
+    for word in word_list[skip_start:]:
+        token_weight = len(word)
+        fun_logger.info("Słowo kluczowe: %s", word)
+        itr = 0
+        for file in music_file_list:
+            file = file.split(SEPARATOR_FILE_PATH)
+
+            char_remove = [
+                ".",
+                "^",
+                "$",
+                "*",
+                "+",
+                "?",
+                "{",
+                "}",
+                "[",
+                "]",
+                "\\",
+                "/",
+                "|",
+                "(",
+                ")",
+                "!",
+                ",",
+                "-",
+                ":",
+                "mp3",
+            ]
+            all_words = []
+            for f_iter in file:
+                for char in char_remove:
+                    f_iter = remove_characters(f_iter, char)
+                tmp = f_iter.split()
+                all_words.extend(tmp)
+            pingu = 1
+            pattern_len = len(all_words)
+            if platform == "win32":
+                skip = 2
+            else:
+                skip = 4
+            matched_times = 1
+            # TODO: To do wątków albo for asynchroniczny
+            for itm in all_words[skip:]:
+                pingu += 1
+                pattern = ".*" + word + ".*"
+                if re.match(pattern, itm, re.IGNORECASE):
+                    temp_weight = (
+                        search_weight[itr][0]
+                        + (token_weight + (pingu**1.5) / pattern_len) / matched_times
+                    )
+                    search_weight[itr] = (
+                        temp_weight,
+                        music_file_list[itr],
+                    )
+                    matched_times += 1
+            itr += 1
+
+    fun_logger.info(
+        "Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start
+    )
+    time_start = datetime.now()
+    item_to_search = max_weight(search_weight)
+    itr = 0
+    if item_to_search == 0:
+        return []
+    not_found = True
+    return_list = []
+    if int(how_many) <= 0:
+        fun_logger.info("Jeden plik do zagrania")
+        while not_found:
+            if search_weight[itr][0] == item_to_search:
+                return_list.append(search_weight[itr])
+                if not return_to_bot:
+                    with open(
+                        "/home/pi/Conjurer/priority_queue.playlist",
+                        "r+",
+                        encoding="utf-8",
+                    ) as s_file:
+                        s_file.write(search_weight[itr][1])
+                break
+            itr += 1
+    else:
+        fun_logger.info("Wiele plików do zagrania")
+        search_weight.sort(key=lambda x: x[0], reverse=True)
+        return_list.extend(search_weight[: int(how_many)])
+    fun_logger.info("Done: %s", return_list)
+    return return_list
+
+
+def max_weight(lista):
+    """
+    Take a list of weights and return the maximum weight.
+
+    :param lista: It seems like the parameter `lista` is a list of items, possibly representing weights.
+    The function name `max_weight` suggests that the function is intended to find the maximum weight
+    from the list. However, without more context or information about the problem, it's difficult to say
+    for sure what the
+    """
+    maximum_weight = 0
+    for iterator in lista:
+        if iterator[0] > maximum_weight:
+            maximum_weight = iterator[0]
+    return maximum_weight
+
+
+def remove_characters(string, character):
+    """
+    The `remove_characters` function removes all occurrences of a specified character from a given
+    string.
+
+    :param string: The string parameter is the input string from which characters will be removed
+    :param character: The character parameter is the character that you want to remove from the string
+    :return: a new string where all occurrences of the specified character have been removed.
+    """
+    return string.replace(character, "")
+
+
+@app.route("/stream", methods=["GET"])
+def stream_music():
+    """
+    The function `stream_music` is a route handler for the "/stream" endpoint.
+    It returns a JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
+
+    :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
+    """
+
+    #return send_from_directory("/tmp/hls", "stream.m3u8")
+    return render_template("/home/pi/Conjurer/stream.html")
+
+@app.route("/<string:file_name>")
+def stream(file_name):
+    """
+    Stream the specified file from the video directory.
+
+    Args:
+        file_name (str): The name of the file to be streamed.
+
+    Returns:
+        Response: The response containing the streamed file.
+    """
+    # trunk-ignore(bandit/B108)
+    video_dir = "/tmp/hls"
+    return send_from_directory(video_dir, file_name)
+
+@app.route("/stream_mp3", methods=["GET"])
+def stream_music_mp3():
+    """
+    The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint.
+    It redirects the user to the URL "http://www.example.com" with a status code of 302.
+
+    :return: A redirect response to "http://www.example.com" with a status code of 302.
+    """
+    return redirect("http://www.example.com", code=302)
+
+
+@app.route("/clear_pr_pls", methods=["GET"])
+def clear_pr_pls():
+    """
+    The function `clear_pr_pls` clears the contents of the priority queue playlist file.
+
+    :return: A JSON response indicating the success of the operation.
+    """
+    app.logger.info("CLEARING PLAYLIST")
+    with open('/home/pi/Conjurer/priority_queue.playlist', 'w', encoding='utf-8') as cleared_pl:
+        cleared_pl.write('')
+
+    return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
+    return return_data, 200
+
+
+@app.route("/mp3", methods=["GET"])
+def get_music_list():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
+    return jsonify({"music_file_list": music_file_list})
+
+
+@app.route("/update_mp3", methods=["POST"])
+def update_music_list():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record["item"])
+    music_file_list.append(record["item"])
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=record),
+        200,
+    )
+    return return_data
+
+
+@app.route("/get_music", methods=["POST"])
+def look_for_playlist():
+    """
+    The function `look_for_playlist` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+
+    :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`,
+             `message`, `statusCode`, and `data`, with values indicating the success of the operation and the
+             data that was received and added to the `music_file_list`. The status code returned is 200,
+             indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True
+    )
+
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
+        200,
+    )
+    return return_data
+
+
+@app.route("/request_radio_file", methods=["POST"])
+def add_request():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    return_data = wyszukaj(
+        record["lista_slow"], 0, app.logger, False
+    )
+
+    with open(
+        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        200,
+    )
+    return return_data
+
+@app.route("/create_priority_playlist", methods=["POST"])
+def create_priority_playlist():
+    """
+    The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+
+    :return: A tuple containing a JSON response and a status code.
+        The JSON response includes keys `isError`,
+        `message`, `statusCode`, and `data`, with values
+        indicating the success of the operation and the
+        data that was received and added to the `music_file_list`.
+        The status code returned is 200,indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
+    )
+    with open(
+        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        200,
+    )
+    return return_data
+
+
+@app.route("/add_to_priority", methods=["POST"])
+def add_to_priority():
+    """
+    The function `add_to_priority` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+
+    :return: A tuple containing a JSON response and a status code.
+        The JSON response includes keys `isError`,
+        `message`, `statusCode`, and `data`, with values
+        indicating the success of the operation and the
+        data that was received and added to the `music_file_list`.
+        The status code returned is 200,indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
+    )
+    with open(
+        "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        200,
+    )
+    return return_data
+
+
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host HOST_ADDRESS
+    and port 5000 using the Waitress WSGI server.
+
+    This function starts the Waitress server and listens for incoming requests
+    on the specified host and port. It uses the `serve` function from the Waitress
+    library to handle the serving process.
+
+    Parameters:
+        None
+
+    Returns:
+        None
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+if __name__ == "__main__":
+    logger = logging.getLogger("conjurer_musician")
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+    handler = handlers.RotatingFileHandler(
+        filename=LOGFILE,
+        encoding=ENCODING,
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
+    )
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    rescan()
+    logger.info("Started")
+    threads = []
+    # threads.append(threading.Thread(target=flask_debug))
+    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=thread_rescan))
+
+    for worker in threads:
+        worker.start()
+
+    time.sleep(60)
+    threads.append(threading.Thread(target=scan_tracks))
+    threads[2].start()
+
+    for worker in threads:
+        worker.join()

conjurer_musician/conjurer_musician.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer_musician
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/conjurer_musician.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

conjurer_musician/music_utils.py

@@ -0,0 +1,26 @@
+# This Python file uses the following encoding: utf-8
+"""This module contains utility scripts used by musi radiostation Conjurer"""
+import time
+import re
+
+NAME = "/home/pi/Conjurer/persistence.log"
+
+def print_top():
+    """
+    Prints the top 65 lines from a file specified by the NAME variable.
+    """
+    while True:
+        line = ""
+        buffer = ""
+        with open(NAME, "r", encoding="utf-8") as f:
+            for _ in range (45):
+                line = f.readline()
+                if re.match(".*mp3", line):
+                    buffer += line
+            print("=====================================")
+            print(buffer)
+            print("=====================================")
+        time.sleep(180)
+
+if __name__ == "__main__":
+    print_top()

conjurer_musician/pl.config

@@ -0,0 +1,14 @@
+$dodaj_do_ulubionych 100 nightwish
+$dodaj_do_ulubionych 100 amon amarth
+$dodaj_do_ulubionych 100 tactical sekt
+$dodaj_do_ulubionych 100 lacuna coil
+$dodaj_do_ulubionych 100 sefa
+$dodaj_do_ulubionych 100 frenchcore
+$dodaj_do_ulubionych 100 peacock
+$dodaj_do_ulubionych 100 vavamuffin
+$dodaj_do_ulubionych 100 abradab
+$dodaj_do_ulubionych 100 eluveitie
+$dodaj_do_ulubionych 100 youtube
+
+$dodaj_do_ulubionych 100 elyose
+$dodaj_do_ulubionych 1 artisans du chaos

conjurer_musician/radio_conjurer.liq

@@ -0,0 +1,169 @@
+# FILEPATH: /home/mtuszowski/conjurer/conjurer_musician/radio_conjurer.liq
+
+# This script sets up a Liquidsoap radio stream with various features and configurations.
+
+# Load icecast credentials from a JSON file
+let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentials.json")
+
+# Enable replaygain metadata processing
+enable_replaygain_metadata()
+
+# Set up a playlog for tracking played tracks
+
+l = playlog(duration = 72000.0, persistency="/home/pi/Conjurer/persistence.log")
+
+# Function to check if a track can be played based on its metadata
+def check(r)
+    m = request.metadata(r)
+    if l.last(m) < 36000. then
+        log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).")
+        false
+    else
+        l.add(m)
+        true
+    end
+end
+
+# Define playlists to be used in the stream
+s1 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/all_playlist.playlist"))
+s2 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/priority_queue.playlist"))
+s3 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/hit.playlist"))
+
+# Create a request queue for user-generated requests
+requests_queue = request.queue()
+
+# Function to process the request queue and add new requests
+def queue_processing()
+    text=file.lines("/home/pi/Conjurer/request.playlist")
+    if text != [] then
+        list.iter(fun(item) -> requests_queue.push.uri(item), text)
+        file.remove("/home/pi/Conjurer/request.playlist")
+        f = file.open("/home/pi/Conjurer/request.playlist", create=true)
+        f.close()
+    end
+end
+
+# Randomly select a playlist with weights
+s4 = random(id="randomizer", weights=[2, 3, 5], [s1, s2, s3])
+
+# Load jingles playlist
+jingles = (playlist(reload_mode="watch", "/home/pi/Conjurer/jingles.playlist"))
+
+# Create the main stream with random playlist and jingles
+s = rotate(id="randomizer", weights=[10, 1], [s4, jingles])
+
+s = crossfade(fade_out=2., fade_in=2., duration=4., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
+
+# Set up an interactive harbor for controlling the stream
+interactive.harbor(port = 9999)
+
+# Set up interactive controls for bass boost
+f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.)
+g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.)
+b = bass_boost(frequency=f, gain=g, s)
+s = add([s, b])
+
+# Set up interactive control for main volume
+a = interactive.float("main_volume", min=0., max=20., 0.4)
+s = compress.multiband.interactive(bands=7, s)
+
+mic_gain = interactive.float("mic_volume", min=0., max=20., 6.)
+# Apply audio processing effects
+tmic = buffer(input.pulseaudio()) # Microphone
+mic = amplify(mic_gain, tmic)
+mic = gate(threshold=-80., range=-120., mic)
+mic = compress(threshold=0., ratio=2.,mic)
+mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic)
+
+s = add([s, mic])
+
+# Apply audio processing effects
+s = nrj(normalize(s))
+s = amplify(a, s)
+
+
+# Skip blank sections in the stream
+s = blank.skip(max_blank=10., s)
+
+# Configure logging settings
+log_to_stdout = true
+log_to_file = true
+logpath = "/home/pi/Conjurer/radio_log.log"
+loglevel = 3
+set("log.stdout", log_to_stdout)
+set("log.level", loglevel)
+
+# Enable logging to file
+set("log.file", log_to_file)
+set("log.file.path", logpath)
+
+# Set up emergency fallback track
+emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
+radio =  fallback(id="switcher2", track_sensitive=false, [s, emergency])
+# Set up an interactive control for skipping tracks
+p = interactive.bool("Skip track", false)
+
+def http_skip(~protocol, ~data, ~headers, uri)=
+    radio.skip()
+    http.response(code=200,data="Skipped")
+end
+harbor.http.register(port=54321,method="GET","/skip", http_skip)
+
+
+# Function to process the request queue and skip the current track if requested
+def check_skip()
+    if p() then
+        log.info("Skipping current track.")
+        radio.skip()
+        p.set(false)
+    end
+end
+
+# Run the queue processing function every 60 seconds
+thread.run(every=60., queue_processing)
+
+
+# Run the check_skip function every 5 seconds
+thread.run(every=15., check_skip)
+
+
+# Enable persistent script parameters
+interactive.persistent("script.params")
+
+# Configure output formats and destinations
+
+output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
+output.pulseaudio(radio)
+#output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
+# Uncomment the following lines to enable additional output formats
+# output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio)
+# output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio)
+# output.icecast(%aac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="aac-stream", radio)
+# output.icecast(%flac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="flac-stream", radio)
+# output.icecast(%vorbis, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="vorbis-stream", radio)
+# output.icecast(%speex, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="speex-stream", radio)
+# output.icecast(%wav, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="wav-stream", radio)
+# output.icecast(%pcm, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="pcm-stream", radio)
+# output.icecast(%raw, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="raw-stream", radio)
+# output.icecast(%s16l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16l-stream", radio)
+# output.icecast(%s16b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16b-stream", radio)
+# output.icecast(%s24l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24l-stream", radio)
+# output.icecast(%s24b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24b-stream", radio)
+# output.icecast(%s32l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32l-stream", radio)
+# output.icecast(%s32b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32b-stream", radio)
+# output.icecast(%s64l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64l-stream", radio)
+# output.icecast(%s64b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64b-stream", radio)
+# output.icecast(%s128l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128l-stream", radio)
+# output.icecast(%s128b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128b-stream", radio)
+# output.icecast(%s256l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256l-stream", radio)
+# output.icecast(%s256b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256b-stream", radio)
+# output.icecast(%s512l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512l-stream", radio)
+# output.icecast(%s512b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512b-stream", radio)
+# output.icecast(%s1024l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024l-stream", radio)
+# output.icecast(%s1024b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024b-stream", radio)
+# output.icecast(%s2048l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048l-stream", radio)
+# output.icecast(%s2048b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048b-stream", radio)
+# output.icecast(%s4096l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096l-stream", radio)
+# output.icecast(%s4096b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096b-stream", radio)
+# output.icecast(%s8192l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192l-stream", radio)
+# output.icecast(%s8192b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192b-stream", radio)

conjurer_musician/radio_service.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=radio_conjurer Liquidsoap daemon
+After=network.target
+Documentation=http://liquidsoap.fm/
+
+[Service]
+User=pi
+WorkingDirectory=/home/pi/Conjurer/
+ExecStart=/home/pi/.opam/4.13.0/bin/liquidsoap /home/pi/Conjurer/radio_conjurer.liq
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+Alias=radio_conjurer-liquidsoap.service

conjurer_musician/requirements_musician.txt

@@ -0,0 +1,3 @@
+flask
+waitress
+sshtunnel

conjurer_musician/stream.html

@@ -0,0 +1,9 @@
+    <html>
+        <head>
+            <title>HTTP Live Streaming Example</title>
+        </head>
+        <body>
+            <video src="/tmp/hls/stream.m3u8" height="300" width="400">
+            </video>
+        </body>
+    </html>

deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+git pull
+cd ..
+cp ./conjurer/bot.py ./Conjurer/
+cp ./conjurer/settings.json ./Conjurer/
+cp ./conjurer/system_gpt_settings.json ./Conjurer/
+cp ./conjurer/install.sh ./Conjurer/
+cp ./conjurer/requirements.txt ./Conjurer/requirements.txt
+sudo systemctl restart conjurer.service

deploy_music.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd ./conjurer
+git pull
+cd ..
+cp ./conjurer/conjurer_musician/conjurer_musician.py ./Conjurer
+sudo systemctl restart conjurer_musician.service

deploy_radio.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd ./conjurer
+git pull
+cd ..
+cp ./conjurer/conjurer_musician/radio_conjurer.liq ./Conjurer
+sudo systemctl restart radio_service.service

install_main_bot.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+sudo apt-get install python3-dev
+cd /home/pi
+mdkir Conjurer
+cd Conjurer
+python3 -m venv /home/pi/Conjurer/env
+cp /home/pi/conjurer/requirements_bot.txt /home/pi/Conjurer/
+source ./env/bin/activate
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_bot.txt
+sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+sed -i '1i\import logging' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py
+
+sed -i '21i\         logger = logging.getLogger("discord")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py
+sed -i '22i\         logger.info("Playlist")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py
+
+deactivate
+sudo cp ./conjurer.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer.service
+sudo systemctl enable conjurer.service

install_music_bot.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+cd /home/pi
+mkdir Conjurer
+cd Conjurer
+python3 -m venv /home/pi/Conjurer/env
+source /home/pi/Conjurer/env/bin/activate
+cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjurer/
+cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
+cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
+touch /home/pi/Conjurer/discord_mus_service.log
+
+touch /home/pi/Conjurer/playlist.json
+touch /home/pi/Conjurer/prio_playlist.json
+
+touch /home/pi/Conjurer/all_playlist.playlist
+touch /home/pi/Conjurer/priority_queue.playlist
+touch /home/pi/Conjurer/hit.playlist
+touch /home/pi/Conjurer/request.playlist
+touch /home/pi/Conjurer/persistence.log
+echo "[]" > /home/pi/Conjurer/persistence.log
+
+touch /home/pi/Conjurer/radio_log.log
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_musician.txt
+deactivate
+sudo cp /home/pi/conjurer/conjurer_musician/conjurer_musician.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service
+
+sudo cp /home/pi/conjurer/conjurer_musician/radio_service.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start radio_service.service
+sudo systemctl enable radio_service.service
+
+
+sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py

mp3_list.service.py

@@ -1,35 +0,0 @@
-from pathlib import Path, PurePath
-from platform import uname
-from sys import platform
-MUSIC_FOLDER = ""
-if platform in ("linux", "linux2"):
-    SEPARATOR_FILE_PATH = "/"
-    if "microsoft-standard" in uname().release:
-        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
-        MUSIC_FOLDER = "/mnt/g/Muzyka/"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
-
-    else:
-        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
-        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
-        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
-        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
-        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
-        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
-        NETRC_FILE = "/home/pi/.netrc"
-        LOGSTORE = "/home/pi/RetroPie/logs/"
-        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
-        DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
-
-music_file_list = []
-for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
-    temp_music_file = mp3_item.as_posix()
-    if platform == "win32":
-        temp_music_file = temp_music_file.replace("/", "\\")
-    music_file_list.append(temp_music_file)
-

requirements_bot.txt

@@ -0,0 +1,19 @@
+discord
+yt_dlp
+spotify_dl
+spotipy
+openai
+eyed3
+numpy
+pdf2image
+PyPDF2
+requests
+spotipy
+tiktoken
+PyNaCl
+flask
+waitress
+clickupython
+assemblyai[extras]
+SpeechRecognition
+git+https://github.com/imayhaveborkedit/discord-ext-voice-recv

restart_radio.sh

@@ -0,0 +1,3 @@
+sshpass -p raspberry ssh pi@192.168.1.15 << 'ENDSSH'
+sudo reboot
+ENDSSH

settings.json

@@ -204,6 +204,13 @@
             false,
             false
         ],
+        "same plusy\\b": [
+            "Jak na cmentarzu Szefie. Jak na cmentarzu",
+            15.0,
+            0,
+            false,
+            false
+        ],
         "kld\\b": [
             "KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.",
             15.0,

spotify_dl/spotify.py

@@ -239,6 +239,8 @@ def get_item_name(sp, item_type, item_id):
         name = sp.album(album_id=item_id).get("name")
     elif item_type == "track":
         name = sp.track(track_id=item_id).get("name")
+    elif item_type == "artist_top_tracks":
+        name = sp.artist_top_tracks(artist_id=item_id).get("name")
     return sanitize(name)
 
 

spotify_dl/youtube.py

@@ -97,8 +97,8 @@ def write_tracks(tracks_file, song_dict):
 def set_tags(temp, filename, kwargs):
     """
      sets song tags after they are downloaded
-    :param temp: contains index used to obtain more info about song being editted
-    :param filename: location of song whose tags are to be editted
+    :param temp: contains index used to obtain more info about song being edited
+    :param filename: location of song whose tags are to be edited
     :param kwargs: a dictionary of extra arguments to be used in tag editing
     """
     song = kwargs["track_db"][int(temp[-1])]

sr_example.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import logging
+
+from discord.ext.voice_recv.sinks import AudioSink
+
+log = logging.getLogger(__name__)
+
+__all__ = [
+    'SpeechRecognitionSink',
+]
+
+try:
+    import discord.ext.speech_recognition as sr  # type: ignore
+except ImportError:
+
+    def SpeechRecognitionSink(**kwargs) -> AudioSink:
+        """A stub for when the SpeechRecognition module isn't found."""
+        raise RuntimeError('The SpeechRecognition module is required to use this sink.')
+
+else:
+    import time
+    import array
+    import asyncio
+    import audioop
+
+    from collections import defaultdict
+
+    from discord.ext.voice_recv.sinks  import SilencePacket
+
+    from typing import TYPE_CHECKING, TypedDict
+
+    if TYPE_CHECKING:
+        from concurrent.futures import Future as CFuture
+        from typing import Literal, Callable, Optional, Any, Final, Protocol, Awaitable, TypeVar
+
+        from discord import Member
+
+        from ..opus import VoiceData
+        from ..types import MemberOrUser as User
+
+        T = TypeVar('T')
+
+        SRRecognizerMethod = Literal[
+            'sphinx',
+            'google',
+            'google_cloud',
+            'wit',
+            'azure',
+            'bing',
+            'lex',
+            'houndify',
+            'amazon',
+            'assemblyai',
+            'ibm',
+            'tensorflow',
+            'whisper',
+            'vosk',
+        ]
+
+        class SRStopper(Protocol):
+            def __call__(self, wait: bool = True, /) -> None:
+                ...
+
+        SRProcessDataCB = Callable[[sr.Recognizer, sr.AudioData, User], Optional[str]]
+        SRTextCB = Callable[[User, str], Any]
+
+    class _StreamData(TypedDict):
+        stopper: Optional[SRStopper]
+        recognizer: sr.Recognizer
+        buffer: array.array[int]
+
+    class SpeechRecognitionSink(AudioSink):  # type: ignore
+        def __init__(
+            self,
+            *,
+            process_cb: Optional[SRProcessDataCB] = None,
+            text_cb: Optional[SRTextCB] = None,
+            default_recognizer: SRRecognizerMethod = 'assemblyai',
+            phrase_time_limit: int = 10,
+            ignore_silence_packets: bool = True,
+        ):
+            super().__init__(None)
+            self.process_cb: Optional[SRProcessDataCB] = process_cb
+            self.text_cb: Optional[SRTextCB] = text_cb
+            self.phrase_time_limmit: int = phrase_time_limit
+            self.ignore_silence_packets: bool = ignore_silence_packets
+
+            self.default_recognizer: SRRecognizerMethod = default_recognizer
+            self._stream_data: defaultdict[int, _StreamData] = defaultdict(
+                lambda: _StreamData(stopper=None, recognizer=sr.Recognizer(), buffer=array.array('B'))
+            )
+
+        def _await(self, coro: Awaitable[T]) -> CFuture[T]:
+            assert self.client is not None
+            return asyncio.run_coroutine_threadsafe(coro, self.client.loop)
+
+        def wants_opus(self) -> bool:
+            return False
+
+        def write(self, user: Optional[User], data: VoiceData) -> None:
+            if self.ignore_silence_packets and isinstance(data.packet, SilencePacket):
+                return
+
+            if user is None:
+                return
+
+            sdata = self._stream_data[user.id]
+            sdata['buffer'].extend(data.pcm)
+
+            if not sdata['stopper']:
+                sdata['stopper'] = sdata['recognizer'].listen_in_background(
+                    DiscordSRAudioSource(sdata['buffer']), self.background_listener(user), self.phrase_time_limmit
+                )
+
+        def background_listener(self, user: User):
+            process_cb = self.process_cb or self.get_default_process_callback()
+            text_cb = self.text_cb or self.get_default_text_callback()
+
+            def callback(_recognizer: sr.Recognizer, _audio: sr.AudioData):
+                output = process_cb(_recognizer, _audio, user)
+                if output is not None:
+                    text_cb(user, output)
+
+            return callback
+
+        def get_default_process_callback(self) -> SRProcessDataCB:
+            def cb(recognizer: sr.Recognizer, audio: sr.AudioData, user: Optional[User]) -> Optional[str]:
+                log.debug("Got %s, %s, %s", audio, audio.sample_rate, audio.sample_width)
+                text: Optional[str] = None
+                try:
+                    # they changed recognize_google to be optionally assigned at runtime...
+                    func = getattr(recognizer, 'recognize_' + self.default_recognizer, recognizer.recognize_google) # type: ignore
+                    text = func(audio)  # type: ignore
+                except sr.UnknownValueError:
+                    log.debug("Bad speech chunk")
+                    # self._debug_audio_chunk(audio)
+
+                return text
+
+            return cb
+
+        def get_default_text_callback(self) -> SRTextCB:
+            def cb(user: Optional[User], text: Optional[str]) -> Any:
+                log.info("%s said: %s", user.display_name if user else 'Someone', text)
+
+            return cb
+
+        @AudioSink.listener()
+        def on_voice_member_disconnect(self, member: Member, ssrc: Optional[int]) -> None:
+            self._drop(member.id)
+
+        def cleanup(self) -> None:
+            for user_id in tuple(self._stream_data.keys()):
+                self._drop(user_id)
+
+        def _drop(self, user_id: int) -> None:
+            data = self._stream_data.pop(user_id)
+
+            stopper = data.get('stopper')
+            if stopper:
+                stopper()
+
+            buffer = data.get('buffer')
+            if buffer:
+                # arrays don't have a clear function
+                del buffer[:]
+
+        def _debug_audio_chunk(self, audio: sr.AudioData, filename: str = 'sound.wav') -> None:
+            import io, wave, discord
+
+            with io.BytesIO() as b:
+                with wave.open(b, 'wb') as writer:
+                    writer.setframerate(48000)
+                    writer.setsampwidth(2)
+                    writer.setnchannels(2)
+                    writer.writeframes(audio.get_wav_data())
+
+                b.seek(0)
+                f = discord.File(b, filename)
+                self._await(self.voice_client.channel.send(file=f))  # type: ignore
+
+    class DiscordSRAudioSource(sr.AudioSource):
+        little_endian: Final[bool] = True
+        SAMPLE_RATE: Final[int] = 48_000
+        SAMPLE_WIDTH: Final[int] = 2
+        CHANNELS: Final[int] = 2
+        CHUNK: Final[int] = 960
+
+        def __init__(self, buffer: array.array[int]):
+            self.buffer = buffer
+            self._entered: bool = False
+
+        @property
+        def stream(self):
+            return self
+
+        def __enter__(self):
+            if self._entered:
+                log.warning('Already entered sr audio source')
+            self._entered = True
+            return self
+
+        def __exit__(self, *exc) -> None:
+            self._entered = False
+            if any(exc):
+                log.exception('Error closing sr audio source')
+
+        def read(self, size: int) -> bytes:
+            # TODO: make this timeout configurable
+            for _ in range(10):
+                if len(self.buffer) < size * self.CHANNELS:
+                    time.sleep(0.1)
+                else:
+                    break
+            else:
+                if len(self.buffer) == 0:
+                    return b''
+
+            chunksize = size * self.CHANNELS
+            audiochunk = self.buffer[:chunksize].tobytes()
+            del self.buffer[: min(chunksize, len(audiochunk))]
+            audiochunk = audioop.tomono(audiochunk, 2, 1, 1)
+            return audiochunk
+
+        def close(self) -> None:
+            self.buffer.clear()

stream_test.py

@@ -0,0 +1,62 @@
+# Start by making sure the `assemblyai` package is installed.
+# If not, you can install it by running the following command:
+# pip install -U assemblyai
+#
+# Then, make sure you have PyAudio installed: https://pypi.org/project/PyAudio/
+#
+# Note: Some macOS users might need to use `pip3` instead of `pip`.
+
+import pyaudio
+import assemblyai as aai
+
+aai.settings.api_key = "aa9962f0088a449a9c4ab2361e96cc08"
+
+def on_open(session_opened: aai.RealtimeSessionOpened):
+  "This function is called when the connection has been established."
+
+  print("Session ID:", session_opened.session_id)
+
+def on_data(transcript: aai.RealtimeTranscript):
+  "This function is called when a new transcript has been received."
+
+  if not transcript.text:
+    return
+
+  if isinstance(transcript, aai.RealtimeFinalTranscript):
+    print(transcript.text, end="\r\n")
+  else:
+    print(transcript.text, end="\r")
+
+def on_error(error: aai.RealtimeError):
+  "This function is called when the connection has been closed."
+
+  print("An error occured:", error)
+
+def on_close():
+  "This function is called when the connection has been closed."
+
+  print("Closing Session")
+
+transcriber = aai.RealtimeTranscriber(
+  on_data=on_data,
+  on_error=on_error,
+  sample_rate=44_100,
+  on_open=on_open, # optional
+  on_close=on_close, # optional
+)
+
+
+pa = pyaudio.PyAudio()
+for i in range(pa.get_device_count()):
+  print (pa.get_device_info_by_index(i))
+
+# Start the connection
+#transcriber.connect()
+
+# Open a microphone stream
+#microphone_stream = aai.extras.MicrophoneStream()
+
+# Press CTRL+C to abort
+#transcriber.stream(microphone_stream)
+
+#transcriber.close()

test_time.py

@@ -0,0 +1,7 @@
+import time
+first_time = time.time_ns()
+
+time.sleep(1)
+time_diff = time.time_ns() - first_time
+print(time_diff)
+#2000149433

utils/cut_large_file.py

@@ -1,9 +1,30 @@
 # trunk-ignore-all(pylint/C0301)
+"""
+The function `cut_my_life_into_pieces` reads a source file, splits its content based on a file size
+limit, and writes the parts to separate output files in a specified folder.
 
-FILE_SIZE_LIMIT = 100000 * 1024
-SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
-OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
-OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
+:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
+content that you want to write to a file. It represents the actual content that will be written to
+the file
+:param file: The code you provided defines two functions: `smart_write` and
+`cut_my_life_into_pieces`. The `smart_write` function encodes input lines to UTF-8, writes them to a
+file, and returns the size of the encoded bytes. The `cut_my_life_into_pieces` function reads
+:return: The functions provided in the code snippet do not explicitly return any values. However,
+the `smart_write` function returns the size of the encoded bytes of the input `lines` in bytes. The
+`cut_my_life_into_pieces` function does not have a return statement, so it does not explicitly
+return any value.
+"""
+
+FILE_SIZE_LIMIT = 50000 * 1024
+
+
+SOURCE = r"C:\Database\dois-2022-02-12\sci-hub-doi-2022-02-12.txt"
+OUTPUT_FOLDER = r"C:\Database\\chunks\\"
+OUTPUT_FILE_SCHEMA = r"_chunk.txt"
+
+#SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
+#OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
+#OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
 
 
 def smart_write(lines, file):
@@ -18,9 +39,12 @@ def smart_write(lines, file):
     :return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
     bytes.
     """
-    encoded_bytes = lines.encode("utf-8")
-    size_in_bytes = len(encoded_bytes)
-    file.write(lines)
+    try:
+        encoded_bytes = lines.encode("utf-8")
+        size_in_bytes = len(encoded_bytes)
+        file.write(lines)
+    except:
+        return 0
 
     return size_in_bytes
 
@@ -44,8 +68,9 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
     with open(source, encoding="utf-8") as infile:
         output_size = 0
         output_no = 0
-
+        lineno = 0
         for line in infile:
+            lineno += 1
             if output_size > FILE_SIZE_LIMIT:
                 output_no += 1
                 output_size = 0
@@ -55,4 +80,12 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
                 "a",
                 encoding="utf-8",
             ) as op_file:
-                output_size += smart_write(line, op_file)
+                tmp =  smart_write(line, op_file)
+                if tmp > 0:
+                    output_size += smart_write(line, op_file)
+                else:
+                    print (f'Error at {lineno}')
+                    print(line)
+
+if __name__ == "__main__":
+    cut_my_life_into_pieces(SOURCE,OUTPUT_FOLDER,OUTPUT_FILE_SCHEMA)

utils/update_memory.py

@@ -0,0 +1,33 @@
+import json
+MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
+ENCODING = "utf-8"
+
+messages = [
+    {"role": "user", "content" : "Hammer: All systems operational"},
+    {"role": "assistant", "content": "Aye, Szefie! Wszystkie systemy działają sprawnie. Gotów do dalszej służby! Co podać?"},
+    {"role": "user", "content" : "Hammer: Podaj coś na orzeźwienie i przebudzenie. COś co rozgrzeje moje stare kości po ciężkiej pracy!"},
+    {"role": "assistant", "content": "Z przyjemnością Szefie! Dla Ciebie przygotuję Midzynarodówkę Interstelarną Wolf's Dragoon - wysoce energetyczny koktajl rozgrzewający i pobudzający jednocześnie."+
+                                    "Do przygotowania potrzebuję:"+
+                                    "\n"+
+                                    "60 ml mocnego rumu z Wolfa (lub innego, jeśli wolisz)"+
+                                    "30 ml soku z cytryny"+
+                                    "35 ml domowego syropu z mięty"+
+                                    "15 ml świeżo zmielonej kawy"+
+                                    "\n"+
+                                    "Mieszam wszystko w shakerze z lodem, a potem przelewam przez sitko do kieliszka koktajlowego. W górę serwuję z bigoskiem z miętą i zestem z cytryny. Rozgrzewa, orzeźwia i daje kopa na resztę nocy."+
+                                    "\n"+
+                                    "Co powiesz Szefie, podać Ci taki?"},
+    {"role": "user", "content" : "Hammer: Brzmi jak coś co zrobi robotę! Skal!"},
+]
+
+
+for temp in messages:
+
+    with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
+        # First we load existing data into a dict.
+        file_data = json.load(file_memory)
+        # Join new_data with file_data inside emp_details
+        file_data.append(temp)
+        file_memory.seek(0)
+        # convert back to json.
+        json.dump(file_data, file_memory, indent=4)

voice_example.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+import discord
+from discord.ext import commands, voice_recv
+
+discord.opus._load_default()
+
+bot = commands.Bot(command_prefix=commands.when_mentioned, intents=discord.Intents.all())
+
+class Testing(commands.Cog):
+    def __init__(self, bot):
+        self.bot = bot
+
+    @commands.command()
+    async def test(self, ctx):
+        def callback(user, data: voice_recv.VoiceData):
+            print(f"Got packet from {user}")
+
+            ## voice power level, how loud the user is speaking
+            # ext_data = packet.extension_data.get(voice_recv.ExtensionID.audio_power)
+            # value = int.from_bytes(ext_data, 'big')
+            # power = 127-(value & 127)
+            # print('#' * int(power * (79/128)))
+            ## instead of 79 you can use shutil.get_terminal_size().columns-1
+
+        vc = await ctx.author.voice.channel.connect(cls=voice_recv.VoiceRecvClient)
+        vc.listen(voice_recv.BasicSink(callback))
+
+    @commands.command()
+    async def stop(self, ctx):
+        await ctx.voice_client.disconnect()
+
+    @commands.command()
+    async def die(self, ctx):
+        ctx.voice_client.stop()
+        await ctx.bot.close()
+
+@bot.event
+async def on_ready():
+    print('Logged in as {0.id}/{0}'.format(bot.user))
+    print('------')
+    await bot.add_cog(Testing(bot))
+
+REMOTE_HOST_NAME = "discord"
+
+
+bot.run("NzUxNDQxODgzNDA2MDA4NDEw.Go--wb.Nr28Oo4eYAP1p2XFVHF5uIQfcPD7s_jO2NfMCQ")

voice_recognition.py

@@ -0,0 +1,233 @@
+import logging
+import wave
+import asyncio
+import assemblyai as aai
+import discord
+from discord.ext import commands, voice_recv, tasks
+from discord.opus import Decoder as OpusDecoder
+import time
+# Replace with your API key
+aai.settings.api_key = "aa9962f0088a449a9c4ab2361e96cc08"
+discord.opus._load_default()
+CHANNELS = OpusDecoder.CHANNELS
+SAMPLE_WIDTH = OpusDecoder.SAMPLE_SIZE // OpusDecoder.CHANNELS
+SAMPLING_RATE = OpusDecoder.SAMPLING_RATE
+
+#rotate file after there is 0.5s between last received pcm for user.
+#delete messsages after user disconnect
+
+logger = logging.getLogger("discord")
+location = "/home/pi/Conjurer/transcripts/"
+
+class CommunicationObject:
+    def __init__(self, msg_type, user, data):
+        self.type = msg_type
+        self.user = user
+        self.data = data
+    def __repr__(self):
+        return f"{self.type} : {self.user} : {self.data}"
+
+class WaveWriter:
+    def __init__(self, user_id, queue):
+        self.queue = queue
+        self.user = user_id
+        self.username = str(user_id.id)
+        #here we can add hashing function to make transcription files not possible to be connected with discord id
+        self.present_file_id = 0
+        self.file_name_past = None
+        self.file_name_present = location + self.username + "_" + str(self.present_file_id) + ".mp3"
+        self.transcript_file_present : wave.Wave_write = wave.open(
+            self.file_name_present, "wb"
+        )
+        self.transcript_file_present.setnchannels(CHANNELS)
+        self.transcript_file_present.setsampwidth(SAMPLE_WIDTH)
+        self.transcript_file_present.setframerate(SAMPLING_RATE)
+        self.file_name_future = location + self.username + "_" + str(self.present_file_id + 1) + ".mp3"
+        self.transcript_file_future : wave.Wave_write = wave.open(
+            self.file_name_future, "wb"
+        )
+        self.transcript_file_future.setnchannels(CHANNELS)
+        self.transcript_file_future.setsampwidth(SAMPLE_WIDTH)
+        self.transcript_file_future.setframerate(SAMPLING_RATE)
+
+    async def rotate(self):
+
+        self.transcript_file_present.close()
+        self.file_name_past = self.file_name_present
+        if self.present_file_id > 10:
+            self.present_file_id = 0
+        else:
+            self.present_file_id += 1
+        self.file_name_present = self.file_name_future
+        self.transcript_file_present = self.transcript_file_future
+        self.file_name_future = location + self.username + "_" + str(self.present_file_id + 1) + ".mp3"
+        self.transcript_file_future : wave.Wave_write = wave.open(
+            self.file_name_future, "wb"
+        )
+        self.transcript_file_future.setnchannels(CHANNELS)
+        self.transcript_file_future.setsampwidth(SAMPLE_WIDTH)
+        self.transcript_file_future.setframerate(SAMPLING_RATE)
+        operation = CommunicationObject(msg_type="send_file",user=self.user,data=[self.file_name_past])
+
+        await self.queue.put(operation)
+
+    def writeframes(self, pcmdata):
+        self.transcript_file_present.writeframes(pcmdata)
+
+    def cleanup(self):
+        logger.info("Cleanup for user %s", self.username)
+        self.transcript_file_present.close()
+        self.transcript_file_future.close()
+
+
+class SRBuffer(voice_recv.AudioSink):
+    """Endpoint AudioSink that generates a wav file.
+    Best used in conjunction with a silence generating sink. (TBD)
+    """
+    # on member join dodajemytypa do listy
+    # on member disconnect - dropujemy go
+    def __init__(self, queue):
+        super().__init__()
+        self.queue = queue
+        self.wavewriter = {}
+
+    def on_user_connect(self, username):
+        self.wavewriter[str(username.id)] = [WaveWriter(username, self.queue), time.time_ns(), time.time_ns(), False, False]
+
+    def on_user_disconnect(self,username):
+        self.wavewriter[str(username.id)].cleanup()
+        self.wavewriter.pop(str(username.id))
+
+    def wants_opus(self) -> bool:
+        return False
+
+    def write(self, user, data) -> None:
+        #logger.info("DAta write")
+        if user:
+            self.wavewriter[str(user.id)][0].writeframes(data.pcm)
+            self.wavewriter[str(user.id)][1] = time.time_ns() #time from last write
+            self.wavewriter[str(user.id)][3] = True #data written in last iter
+            self.wavewriter[str(user.id)][4] = True #data in buff
+            time.sleep(0.001)
+
+    def cleanup(self) -> None:
+        try:
+            logger.info("Cleanup for SRBuffer")
+            for item in self.wavewriter.values():
+                item[0].cleanup()
+        except Exception:
+            logger.warning("WaveSink got error closing file on cleanup", exc_info=True)
+
+
+class Transcriber(commands.Cog):
+    def __init__(self, bot):
+        self.bot = bot
+        self.threads = []
+        self.comm_queue = asyncio.Queue()
+        self.wsink = SRBuffer(self.comm_queue)
+        self.worker = None
+        self.mc = None
+        self.vc = None
+
+    @commands.hybrid_command(name="transcribe")
+    async def test(self, ctx):
+        if self.vc:
+            vc = self.vc #to juz powinien byc voice channel z funkcja conenct
+        else:
+            vc = None
+        self.mc = ctx.message.channel
+        if self.bot.voice_clients:
+            if  isinstance(self.bot.voice_clients[0], voice_recv.VoiceRecvClient):
+                logger.debug("Already transcribing")
+            else:
+                logger.debug("Already connected with other client")
+        else:
+            logger.debug("Connected")
+            vc = await ctx.author.voice.channel.connect(cls=voice_recv.VoiceRecvClient)
+            logger.info(self.bot.voice_clients)
+            self.check_data.start()
+            self.worker = asyncio.create_task(self.transcribe_output_queue())
+            vc.listen(self.wsink)
+
+
+    @tasks.loop(seconds=0.5)
+    async def check_data(self):
+        for item in self.wsink.wavewriter.values():
+            if not item[3]:
+                timediff_rotation = time.time_ns() - item[2]
+                timediff_write = time.time_ns() -item[1]
+                if  timediff_rotation > 13000149433 and timediff_write > 500014943 and item[4]:
+                    logger.info("File rotation time since last write %s %s", timediff_write, timediff_write/1e9)
+                    logger.info("File rotation time since last rotation %s %s",timediff_rotation, timediff_rotation/1e9)
+                    item[2] = time.time_ns()
+                    item[4] = False
+                    await item[0].rotate()
+            item[3] = False
+
+    @commands.Cog.listener()
+    async def on_voice_state_update(self, user, before, after):
+        if user == self.bot.user:
+            logger.debug("Ignoring self")
+            return
+
+        if before.channel is None:
+            logger.info("User %s connected to channel %s", user, after.channel.name)
+            self.wsink.on_user_connect(user)
+        elif after.channel is  None:
+            logger.info("User %s disconnected from channel %s", user, before.channel.name)
+            operation = CommunicationObject(msg_type="user_cleanup", user=user, data=None)
+            await self.comm_queue.put(operation)
+        else:
+            logger.debug("User VC status changed %s", user.id)
+            logger.debug("Before %s", before)
+            logger.debug("After %s", after)
+
+
+    @commands.command(name="stop_transcribe")
+    async def stop(self, ctx):
+        self.check_data.stop()
+        stop_token = CommunicationObject("STOP",None,None)
+
+        await self.comm_queue.put(stop_token)
+        await ctx.voice_client.disconnect()
+
+    async def transcribe_output_queue(self):
+        logger.info("Transcript uploader start")
+        config = aai.TranscriptionConfig(language_code="pl")
+        transcriber = aai.Transcriber()
+        logger.debug("Transcriber queue id: %s",id(self.comm_queue))
+
+        while True:
+            logger.debug("waiting for tasks")
+            item = await self.comm_queue.get()
+            logger.debug("Got %s", item)
+            if "STOP" in item.type:
+                logger.debug("Queue ended")
+                break
+            elif "send_file" in item.type:
+                logger.debug("Sending file for transcription")
+                transcript = None
+                for iter in item.data:
+                    try:
+                        coro = asyncio.to_thread(
+                            transcriber.transcribe,
+                            iter,
+                            config=config
+                        )
+                        transcript = await coro
+                    except Exception as e:
+                        logger.warn("Exceptiom occured %s", e)
+                    await self.mc.send(f"{item.user} : {transcript.text}")
+                    if transcript.error:
+                        logger.error(transcript.error)
+                        raise AssertionError
+            elif "user_cleanup" in item.type:
+                logger.info("User %s disconnected - cleanup action")
+            else:
+                logger.warn("Something went wrong with object %s", item)
+        logger.info("Transcript uploader stoped")
+
+async def setup(bot):
+    logger = logging.getLogger("discord")
+    await bot.add_cog(Transcriber(bot))
+    logger.info("Loading voice transcribed module done")