Bugfix in script deploy_music.sh

Adding playlists, various bufixes and installation scripts

Fixing

Rename

KUUUURWA

Install script music bot

fx

Fix ?

fix

Synchro

test

test

Undo

tst

test

tst

Undo changes

test

fx

tst

fx

Test commit of two files priority queue.

communication_subroutine.py

@@ -0,0 +1,76 @@
+import logging
+import threading
+import json
+from queue import Queue, Empty
+from flask import Flask, jsonify, request
+from waitress import serve
+
+HOST_ADDRESS = "192.168.1.191"
+PORT_ADDRESS = 5000
+OUT_COMM_Q = Queue()
+IN_COMM_Q = Queue()
+internal_q = Queue()
+app = Flask(__name__)
+
+@app.route("/conjurer", methods=["POST"])
+def answer_external_command():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info("DATA RECEIVED")
+
+    
+    while True:
+        try:
+            q_data = internal_q.get(block=False)
+        except Empty:
+            break
+    if q_data.uuid in record:
+        pass
+
+    return jsonify("SUCCESS")
+
+@app.route("/conjurer", methods=["GET"])
+def check_alive():
+    return jsonify("ALIVE")
+
+def flask_debug(_logger):
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    _logger.info("Attempt debug")
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+def waitress_run(_logger):
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    _logger.info("Attempt waitress")
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def scan_queue(_logger):
+    while True:
+        data = OUT_COMM_Q.get()
+        _logger.info(data)
+        internal_q.put(data)
+        query_list.append(data)
+
+def comm_subroutine(logger):
+    #logger.setLevel(logging.DEBUG)
+    logger.info("Started")
+    threads = []
+    threads.append(threading.Thread(target=flask_debug, args=(logger,)))
+    #threads.append(threading.Thread(target=waitress_run, args=(logger,)))
+    threads.append(threading.Thread(target=scan_queue,args=(logger,)))
+
+    for worker in threads:
+        worker.start()
+    for worker in threads:
+        worker.join()
+
+if __name__ == "__main__":
+    logging_client = logging.getLogger(__name__)
+    comm_subroutine(logging_client)

conjurer.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/bot.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

conjurer_librarian/conjurer_librarian.py

@@ -0,0 +1,342 @@
+import asyncio
+import json
+import logging
+import netrc
+import re
+import threading
+import time
+from json.decoder import JSONDecodeError
+from queue import Queue
+from typing import Callable
+from urllib.request import urlopen
+from uuid import UUID
+
+import requests
+import search_bot
+from flask import Flask, jsonify, request
+from habanero import Crossref
+from waitress import serve
+
+NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
+
+HOST_ADDRESS = "192.168.1.192"
+PORT_ADDRESS = 5001
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+SEND_RESULTS = "/conjurer"
+BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
+
+ENCODING = "utf-8"
+app = Flask(__name__)
+
+librarian_queue = Queue()
+librarian_list = []
+
+
+class Librarian(object):
+
+    def __init__(self, app, query, uuid) -> None:
+        netrc_mod = netrc.netrc(NETRC_FILE)
+        authTokens = netrc_mod.authenticators("crossref")
+        self.cr = Crossref(mailto=authTokens[0])
+        self.query = query
+        self.uuid = str(uuid)
+        self.limit = 3000
+        self.fetched = 0
+        self.hit = 0
+        self.total = 0
+        self.app = app
+        self.logger = app.logger
+        self.live_results = []
+        self.search_result_from_cr = {}
+        self.done = False
+
+    async def search_crossref(self, query):
+        self.app.logger.info("STARTED SEARCH")
+        result = self.cr.works(query=query, limit=1000)
+        self.search_result_from_cr.update(result)
+        self.total = result["message"]["total-results"]
+        self.fetched += len(result["message"]["items"])
+        self.logger.info(self.total)
+        self.logger.info(self.fetched)
+        while self.total > self.fetched and self.limit > self.fetched:
+            result = self.cr.works(query=query, limit=1000)
+            self.search_result_from_cr.update(result)
+            self.total = result["message"]["total-results"]
+            self.fetched += len(result["message"]["items"])
+            self.logger.info(self.total)
+            self.logger.info(self.fetched)
+
+            time.sleep(1)
+        self.app.logger.info("CROSSREF DONE")
+        # TODO: add to sql with UUID
+        # sql row - title, doi, author, uuid, last retrieved, is_available
+
+        with open("rr_results.json", "r+", encoding="utf-8") as data_file:
+            # First we load existing data into a dict.
+            try:
+                file_data = json.load(data_file)
+            except JSONDecodeError:
+                file_data = {}
+            summarized_results = []
+            self.app.logger.info("REFINE: Removing all derived works from the list")
+            for item in self.search_result_from_cr["message"]["items"]:
+                container = "NO"
+                if "container-title" in item:
+                    container = "YES"
+                else:
+                    summarized_results.append(
+                        {
+                            "DOI": item["DOI"],
+                            "title": item["title"] if "title" in item else None,
+                            "type": item["type"] if "type" in item else None,
+                            "container": container,
+                        }
+                    )
+            result = {
+                self.uuid: {
+                    "total_results": self.search_result_from_cr["message"][
+                        "total-results"
+                    ],
+                    "on_page": 1,
+                    "summary": summarized_results,
+                    "results": self.search_result_from_cr["message"]["items"],
+                }
+            }
+            if file_data:
+                file_data.update(result)
+            else:
+                file_data = result
+
+            self.app.logger.info("REFINE: Dumping to file")
+            data_file.truncate(0)
+            data_file.seek(0)
+            json.dump(file_data, data_file, indent=4)
+            return result
+
+    async def check_dois_in_local_db(self, doi):
+        self.app.logger.info("REFINE: Running search in the backend app")
+        coro = asyncio.to_thread(
+            search_bot.search_for_doi, doi, self.live_results, self.app.logger
+        )
+        result = await coro
+        return result
+
+    async def check_if_exists_brute_force(self, page_url):
+        self.app.logger.error("REFINE: Brute force search!")
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        # trunk-ignore(bandit/B310)
+        with urlopen(page_url) as response:
+            data = response.read()
+            text = data.decode("utf-8")
+            for line in text.splitlines():
+                if re.match(
+                    r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
+                    line,
+                ):
+                    return False
+                if m := re.match(
+                    r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
+                ):
+                    direct_download_link = "https:" + str(m.group(1))
+                    self.app.logger.info(direct_download_link)
+                    return True
+        return False
+
+    async def check_if_exists(self, doi, brute_force=False):
+        result = {}
+        result = await self.check_dois_in_local_db(doi)
+        if brute_force and not result:
+            for item in doi:
+                item_link = "https://sci-hub.se/" + item[0]
+                tmp = {
+                    "DOI": item["DOI"],
+                    "exists": await self.check_if_exists_brute_force(item_link),
+                    "data": item,
+                }
+                result.update(tmp)
+        return result
+
+    async def refine_query(self, unrefined_reqult, brute_force=False):
+        temp = []
+        refined_result = []
+        for key in unrefined_reqult:
+            self.logger.info("KEY:")
+            self.logger.info(key)
+
+        for item in unrefined_reqult[self.uuid]["summary"]:
+            if item["container"] == "NO" and item["title"]:
+                refined_result.append(item)
+
+        for item in refined_result:
+            temp.append([item["DOI"], item])
+
+        refined_result = await self.check_if_exists(temp)
+        temp = []
+        for item in refined_result:
+            if item["exists"]:
+                temp.append(item["data"])
+        refined_result = temp
+        self.hit = len(refined_result)
+        return refined_result
+
+    # TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
+    async def answer_query(self):
+        # TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
+        # Search sql for UUID
+        # SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        self.logger.info(f"Search started {self.uuid}")
+        uuid_found = False
+        database = None
+        query = self.query
+        with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
+            # tu logika komunikacji z SQL i cała logika związana z wyszukaniem
+            # ale tymczasowo plik
+            try:
+                database = json.load(cr_file)
+            except JSONDecodeError:
+                pass
+
+            if database:
+                for item in database.keys():
+                    if item in self.uuid:
+                        self.result = await self.refine_query(database)
+                        return self.result
+                answer = await self.search_crossref(query=query)
+                database.update(answer)
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database, cr_file)
+                self.result = await self.refine_query(answer)
+                return self.result
+            else:
+                answer = await self.search_crossref(query=query)
+                database = answer
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database, cr_file)
+                self.result = await self.refine_query(answer)
+                self.app.logger.info("Refined")
+                self.done = True
+                return self.result
+
+
+# ============================= FLASK INTERNALS===============================
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+class BackgroundTaskSearch(threading.Thread):
+    def run(self):
+        loop = asyncio.new_event_loop()  # loop = asyncio.get_event_loop()
+        loop.run_until_complete(self._run())
+        loop.close()
+
+    async def _run(self):
+        while True:
+            database = None
+            librarian = librarian_queue.get()
+            self.app.logger.info("STARTED")
+            result = await librarian.answer_query()
+            result = {librarian.uuid: result}
+            self.app.logger.info("Saving to file")
+            with open("s_results.json", "r+", encoding="utf-8") as s_file:
+                try:
+                    database = json.load(s_file)
+                except JSONDecodeError:
+                    pass
+                if database:
+                    database.extend(result)
+                else:
+                    database = result
+                self.app.logger.info("DUMPING DATA")
+                s_file.truncate(0)
+                s_file.seek(0)
+                json.dump(database, s_file)
+            self.app.logger.info("FINISHED")
+
+            self.app.logger.info(result)
+            coroutine = asyncio.to_thread(
+                requests.post,
+                f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+                json=result,
+                timeout=360,
+            )
+            self.app.logger.info("SENT")
+            _ = await coroutine
+            self.app.logger.info("SEND CONFIRMED")
+            time.sleep(1)
+
+
+# ==================================SERVER ROUTES==========================================
+@app.route("/query", methods=["POST"])
+async def query_database():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["query"])
+    app.logger.info(record["UUID"])
+    uuid = record["UUID"]
+    # TODO: TESTING ONLY======================
+    uuid = UUID("{12345678-1234-5678-1234-567812345678}")
+
+    cl = Librarian(app, record["query"], uuid)
+    librarian_queue.put(cl)
+    librarian_list.append(cl)
+    answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
+        200,
+    )
+    return return_data
+
+
+@app.route("/get_partial_result", methods=["POST"])
+async def get_partial():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["UUID"])
+    for lib in librarian_list:
+        if lib.uuid == record["UUID"]:
+            answer_data = lib.live_results
+            break
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
+        200,
+    )
+    return return_data
+
+
+# =======================================MAIN===================================================
+if __name__ == "__main__":
+    logger = logging.getLogger("conjurer_musician")
+    logger.setLevel(logging.DEBUG)
+    threads = []
+    # threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=flask_debug))
+    bgtask = BackgroundTaskSearch()
+    bgtask.app = app
+    threads.append(bgtask)
+    i = 0
+    for worker in threads:
+        try:
+            logger.info(f"App number: {i}")
+            i += 1
+            worker.start()
+        except RuntimeError as e:
+            logger.error("Exploded")
+            print(str(e))
+    for worker in threads:
+        worker.join()

conjurer_librarian/install_librarian_service.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+python3 -m venv /home/pi/Conjurer_librarian/env
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+cp  /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
+cd /home/pi/Conjurer_librarian
+source /home/pi/Conjurer_librarian/env/bin/activate
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_librarian.txt
+deactivate
+
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/

conjurer_librarian/requirements_librarian.txt

@@ -0,0 +1,3 @@
+habanero
+waitress
+flask

conjurer_librarian/search_bot.py

@@ -0,0 +1,106 @@
+from queue import Queue, Empty
+from threading import Thread
+q = Queue()
+
+#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy
+#MAXTHREADS = 3
+MAXTHREADS = 1
+
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+DATABASE_PATH = r'C:\\Database\\chunks\\'
+
+ENCODING = "utf-8"
+CHUNk = "_chunk.txt"
+_sentinel = object()
+result_list = []
+
+#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+
+def producer(out_q, control_q, filename):
+    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+        lines = operated_file.readlines()
+        for line in lines:
+            out_q.put(line)
+            try:
+                check = control_q.get(block=False)
+            except Empty:
+                check = False
+                pass
+            if check is _sentinel:
+                print("TERM signal received")
+                control_q.put(check)
+                break
+    print(f"Worker finished: {filename}")
+    out_q.put(_sentinel)
+ 
+#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
+def consumer(in_q, control_q, doi, live_results, logger):
+    sentinels = 0
+    for item in doi:
+        result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
+    while True:
+        #if in_q.empty():
+        #    print("Queue empty!")
+        #if in_q.full():
+        #    print("Queue full!")
+        done_check = True
+        data = in_q.get()
+        if data is _sentinel:
+            logger.info("Worker finished")
+            sentinels += 1
+        else:
+            for item in result_list:
+                if item["DOI"] in data:
+                    logger.info(data)
+                    logger.info("HIT")
+                    item["exists"] = True
+                    live_results.append(item)
+                done_check = done_check and item["exists"]
+            if done_check:
+                control_q.put(_sentinel)
+        if sentinels >= MAXTHREADS:
+            logger.info("All workers finished")
+            break
+
+
+def search_for_doi(doi, live_results, logger):
+    threads = []
+    work_q = Queue(maxsize=200000)
+    control_q = Queue()
+    t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
+    logger.info("Consumer thread created")
+    threads.append(t_cons)
+    for i in range(0,MAXTHREADS):
+        #filename = "test" + str(i) + CHUNk
+        filename = "40" + CHUNk
+        #filename = str(i) + CHUNk
+        logger.info(f"Creating worker thread no: {i}")
+        threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
+    for worker in threads:
+        worker.start()
+    for worker in threads:
+        worker.join()
+    return result_list
+
+
+
+if __name__ == "__main__":
+    import logging
+    logger = logging.getLogger()
+    logger.setLevel("DEBUG")
+    h1 = logging.StreamHandler()
+    logger.addHandler(h1)
+    logger.info("TEST RUN")
+    live_result = []
+    logger.info(search_for_doi([
+        ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
+        ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
+        ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
+        ("10.2165/00128415-200309690-00017","DATA"),
+        ("10.5772/48313","DATA"),
+        ("10.15803/ijnc.7.2_419","DATA"),
+        ("10.1051/0004-6361/201321596e","DATA"),
+        ("10.2307/40835941","DATA"),
+        ], live_result, logger))
+    logger.info(live_result)

conjurer_musician/conjurer_musician.py

@@ -0,0 +1,418 @@
+# This Python file uses the following encoding: utf-8
+"""
+The provided Python script sets up a Flask web server to manage a list of music files, with
+functions for rescanning the music folder, updating the music list, and serving the music list via
+API endpoints.
+"""
+import json
+import logging
+import re
+import threading
+import time
+from datetime import datetime
+from logging import handlers
+from pathlib import Path
+from platform import uname
+from sys import platform
+
+from flask import Flask, jsonify, redirect, request, send_from_directory
+from waitress import serve
+
+HOST_ADDRESS = "192.168.1.15"
+PORT_ADDRESS = 5000
+if platform in ("linux", "linux2"):
+    SEPARATOR_FILE_PATH = "/"
+    if "microsoft-standard" in uname().release:
+        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
+        MUSIC_FOLDER = "/mnt/g/Muzyka/"
+        NETRC_FILE = "/home/mtuszowski/.netrc"
+        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
+        ENCODING = "utf-8"
+
+    else:
+        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
+        NETRC_FILE = "/home/pi/.netrc"
+        LOGSTORE = "/home/pi/RetroPie/logs/"
+        ENCODING = "utf-8"
+        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
+
+
+music_file_list = []
+
+
+def create_playlist():
+    """
+    Reads a JSON file containing a playlist and writes the playlist items to a text file.
+
+    The JSON file path is '/home/pi/Conjurer/playlist.json',
+    and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
+
+    Raises:
+        JSONDecodeError: If the JSON file cannot be decoded.
+
+    """
+    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
+        with open(
+            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
+        ) as w_file:
+            try:
+                playlist = json.load(r_file)
+                for item in playlist:
+                    w_file.write(item)
+                    w_file.write("\n")
+            except json.JSONDecodeError:
+                pass
+
+
+def rescan():
+    """
+    The `rescan` function logs a message, scans for mp3 files in a specified folder,
+    and adds them to a list of music files.
+    """
+    logging.info("Rescan triggered")
+    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+        temp_music_file = mp3_item.as_posix()
+        if platform == "win32":
+            temp_music_file = temp_music_file.replace("/", "\\")
+        music_file_list.append(temp_music_file)
+    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
+        database = music_file_list
+        # self.app.logger.info("DUMPING DATA")
+        s_file.truncate(0)
+        s_file.seek(0)
+        json.dump(database, s_file)
+    create_playlist()
+
+
+def thread_rescan():
+    """
+    The `thread_rescan` function periodically triggers a rescan operation after a specified time
+    interval.
+    """
+    logging.info("Starting filesystemupdater")
+    while True:
+        time.sleep(60 * 60 * 24)
+        logging.info("Rescan triggered")
+        rescan()
+
+
+app = Flask(__name__)
+
+
+# TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
+def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
+    """
+    Searches for music files based on a list of keywords and returns a list of matching files.
+
+    Args:
+        word_list (list): A list of keywords to search for.
+        how_many (int): The maximum number of matching files to return.
+
+    Returns:
+        list: A list of matching files, sorted by their search weight.
+
+    """
+    if _logger is None:
+        fun_logger = logging.getLogger()
+    else:
+        fun_logger = _logger
+    return_list = []
+    search_weight = []
+
+    for _ in range(len(music_file_list)):
+        search_weight.append((0, ""))
+
+    time_start = datetime.now()
+    if int(how_many) > 0:
+        skip_start = 2
+    else:
+        skip_start = 1
+    for word in word_list[skip_start:]:
+        token_weight = len(word)
+        fun_logger.info("Słowo kluczowe: %s", word)
+        itr = 0
+        for file in music_file_list:
+            file = file.split(SEPARATOR_FILE_PATH)
+
+            char_remove = [
+                ".",
+                "^",
+                "$",
+                "*",
+                "+",
+                "?",
+                "{",
+                "}",
+                "[",
+                "]",
+                "\\",
+                "/",
+                "|",
+                "(",
+                ")",
+                "!",
+                ",",
+                "-",
+                ":",
+                "mp3",
+            ]
+            all_words = []
+            for f_iter in file:
+                for char in char_remove:
+                    f_iter = remove_characters(f_iter, char)
+                tmp = f_iter.split()
+                all_words.extend(tmp)
+            pingu = 1
+            pattern_len = len(all_words)
+            if platform == "win32":
+                skip = 2
+            else:
+                skip = 4
+            matched_times = 1
+            # TODO: To do wątków albo for asynchroniczny
+            for itm in all_words[skip:]:
+                pingu += 1
+                pattern = ".*" + word + ".*"
+                if re.match(pattern, itm, re.IGNORECASE):
+                    temp_weight = (
+                        search_weight[itr][0]
+                        + (token_weight + (pingu**1.5) / pattern_len) / matched_times
+                    )
+                    search_weight[itr] = (
+                        temp_weight,
+                        music_file_list[itr],
+                    )
+                    matched_times += 1
+            itr += 1
+
+    fun_logger.info(
+        "Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start
+    )
+    time_start = datetime.now()
+    item_to_search = max_weight(search_weight)
+    itr = 0
+    if item_to_search == 0:
+        return []
+    not_found = True
+    return_list = []
+    if int(how_many) <= 0:
+        fun_logger.info("Jeden plik do zagrania")
+        while not_found:
+            if search_weight[itr][0] == item_to_search:
+                return_list.append(search_weight[itr])
+                if not return_to_bot:
+                    with open(
+                        "/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
+                    ) as s_file:
+                        s_file.write(search_weight[itr][1])
+                break
+            itr += 1
+    else:
+        fun_logger.info("Wiele plików do zagrania")
+        search_weight.sort(key=lambda x: x[0], reverse=True)
+        return_list.extend(search_weight[: int(how_many)])
+        if not return_to_bot:
+
+            with open(
+               "/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
+            ) as s_file:
+                s_file.truncate(0)
+                for item in return_list:
+                    s_file.write(item[1] + "\n")
+    return return_list
+
+
+def max_weight(lista):
+    """
+    Take a list of weights and return the maximum weight.
+
+    :param lista: It seems like the parameter `lista` is a list of items, possibly representing weights.
+    The function name `max_weight` suggests that the function is intended to find the maximum weight
+    from the list. However, without more context or information about the problem, it's difficult to say
+    for sure what the
+    """
+    maximum_weight = 0
+    for iterator in lista:
+        if iterator[0] > maximum_weight:
+            maximum_weight = iterator[0]
+    return maximum_weight
+
+
+def remove_characters(string, character):
+    """
+    The `remove_characters` function removes all occurrences of a specified character from a given
+    string.
+
+    :param string: The string parameter is the input string from which characters will be removed
+    :param character: The character parameter is the character that you want to remove from the string
+    :return: a new string where all occurrences of the specified character have been removed.
+    """
+    return string.replace(character, "")
+
+
+@app.route("/stream", methods=["GET"])
+def stream_music():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
+    return send_from_directory("/tmp/hls", "stream.m3u8")
+
+@app.route('/<string:file_name>')
+def stream(file_name):
+    video_dir = '/tmp/hls'
+    return send_from_directory(video_dir, file_name)
+
+
+@app.route("/stream_mp3", methods=["GET"])
+def stream_music_mp3():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
+    return redirect("http://www.example.com", code=302)
+
+
+@app.route("/clear_pr_pls", methods=["GET"])
+def clear_pr_pls():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
+    with open(
+        "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
+        ) as s_file:
+        s_file.truncate(0)
+    return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
+    return return_data, 200
+
+
+
+@app.route("/mp3", methods=["GET"])
+def get_music_list():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
+    return jsonify({"music_file_list": music_file_list})
+
+
+@app.route("/update_mp3", methods=["POST"])
+def update_music_list():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record["item"])
+    music_file_list.append(record["item"])
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=record),
+        200,
+    )
+    return return_data
+
+
+@app.route("/get_music", methods=["POST"])
+def look_for_playlist():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True)
+
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
+        200,
+    )
+    return return_data
+
+@app.route("/add_to_priority", methods=["POST"])
+def add_to_priority():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False)
+
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
+        200,
+    )
+    return return_data
+
+
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+
+if __name__ == "__main__":
+    logger = logging.getLogger("conjurer_musician")
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+    handler = handlers.RotatingFileHandler(
+        filename=LOGFILE,
+        encoding=ENCODING,
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
+    )
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    rescan()
+    logger.info("Started")
+    threads = []
+    #threads.append(threading.Thread(target=flask_debug))
+    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=thread_rescan))
+
+    for worker in threads:
+        worker.start()
+
+    for worker in threads:
+        worker.join()

conjurer_musician/conjurer_musician.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer_musician
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/conjurer_musician.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

conjurer_musician/radio_conjurer.liq

@@ -0,0 +1,48 @@
+let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentials.json")
+enable_replaygain_metadata()
+
+s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist"))
+s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist"))
+
+s = random(id="randomizer", weights=[7000, 100], [s2, s1])
+
+interactive.harbor(port = 9999)
+f = interactive.float("f", description="Frequency", min=0., max=1000.,unit="Hz", 200.)
+g = interactive.float("g", description="Gain", min=0., max=20.,unit="dB", 8.)
+b = bass_boost(frequency=f, gain=g, s)
+s = add([s, b])
+a = interactive.float("main_volume", min=0., max=20., 1.)
+s = compress.multiband.interactive(bands=3, s)
+
+s = nrj(normalize (s))
+s = crossfade(fade_out=3., fade_in=3., duration=5., s)
+s = amplify(a,s)
+
+s = blank.skip(max_blank=2., s)
+
+emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
+radio = fallback(id="switcher",track_sensitive = false, [s, emergency]) 
+
+handle_metadata = fun (m) -> begin
+    print(m["title"])
+    print(m["artist"])
+    print(m["filename"])
+end
+radio.on_metadata(handle_metadata)
+interactive.persistent("script.params")
+
+log_to_stdout = true
+log_to_file = true
+logpath = "/home/pi/Conjurer/radio_log.log"
+loglevel = 3
+
+# Enable logging on Standard Output and set logging level.
+set("log.stdout",log_to_stdout)
+set("log.level",loglevel)
+
+# Enable logging to file. See variable: logpath
+set("log.file",log_to_file)
+set("log.file.path",logpath)
+
+output.file.hls("/tmp/hls",[("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))],radio)
+output.icecast(%mp3, host="retropie", port=8000,password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)

conjurer_musician/radio_service.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=radio_conjurer Liquidsoap daemon
+After=network.target
+Documentation=http://liquidsoap.fm/
+
+[Service]
+User=pi
+WorkingDirectory=/home/pi/Conjurer/
+ExecStart=/home/pi/.opam/4.13.0/bin/liquidsoap /home/pi/Conjurer/radio_conjurer.liq
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+Alias=radio_conjurer-liquidsoap.service

conjurer_musician/requirements_musician.txt

@@ -0,0 +1,2 @@
+flask
+waitress

deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+git pull
+cd ..
+cp ./conjurer/bot.py ./Conjurer/
+cp ./conjurer/settings.json ./Conjurer/
+cp ./conjurer/system_gpt_settings.json ./Conjurer/
+cp ./conjurer/install.sh ./Conjurer/
+cp ./conjurer/requirements.txt ./Conjurer/requirements.txt
+sudo systemctl restart conjurer.service

deploy_music.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd ./conjurer
+git pull
+cd ..
+cp ./conjurer/conjurer_musician/conjurer_musician.py ./Conjurer
+sudo systemctl restart conjurer_musician.service

deploy_radio.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd ./conjurer
+git pull
+cd ..
+cp ./conjurer/conjurer_musician/radio_conjurer.liq ./Conjurer
+sudo systemctl restart radio_service.service

install_main_bot.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+cd /home/pi
+mdkir Conjurer
+cd Conjurer
+python3 -m venv /home/pi/Conjurer/env
+cp /home/pi/conjurer/requirements_bot.txt /home/pi/Conjurer/
+source ./env/bin/activate
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_bot.txt
+sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+deactivate
+sudo cp ./conjurer.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer.service
+sudo systemctl enable conjurer.service

install_music_bot.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+cd /home/pi
+mkdir Conjurer
+cd Conjurer
+python3 -m venv /home/pi/Conjurer/env
+source /home/pi/Conjurer/env/bin/activate
+cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjurer/
+cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
+cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
+touch /home/pi/Conjurer/discord_mus_service.log
+touch /home/pi/Conjurer/all_playlist.txt
+touch /home/pi/Conjurer/playlist.json
+touch /home/pi/Conjurer/priority_queue.playlist
+touch /home/pi/Conjurer/radio_log.log
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_musician.txt
+deactivate
+sudo cp /home/pi/conjurer/conjurer_musician/conjurer_musician.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service
+
+sudo cp /home/pi/conjurer/conjurer_musician/radio_service.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start radio_service.service
+sudo systemctl enable radio_service.service

mp3_list.service.py

@@ -1,35 +0,0 @@
-from pathlib import Path, PurePath
-from platform import uname
-from sys import platform
-MUSIC_FOLDER = ""
-if platform in ("linux", "linux2"):
-    SEPARATOR_FILE_PATH = "/"
-    if "microsoft-standard" in uname().release:
-        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
-        MUSIC_FOLDER = "/mnt/g/Muzyka/"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
-
-    else:
-        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
-        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
-        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
-        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
-        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
-        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
-        NETRC_FILE = "/home/pi/.netrc"
-        LOGSTORE = "/home/pi/RetroPie/logs/"
-        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
-        DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
-
-music_file_list = []
-for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
-    temp_music_file = mp3_item.as_posix()
-    if platform == "win32":
-        temp_music_file = temp_music_file.replace("/", "\\")
-    music_file_list.append(temp_music_file)
-

requirements_bot.txt

@@ -0,0 +1,15 @@
+discord
+yt_dlp
+spotify_dl
+spotipy
+openai
+eyed3
+numpy
+pdf2image
+PyPDF2
+requests
+spotipy
+tiktoken
+PyNaCl
+flask
+waitress

s_results.json

@@ -0,0 +1 @@
+{"12345678-1234-5678-1234-567812345678": [{"DOI": "10.1057/9781137435026", "title": ["Queer BDSM Intimacies"], "type": "book", "container": "NO"}]}

spotify_dl/youtube.py

@@ -97,8 +97,8 @@ def write_tracks(tracks_file, song_dict):
 def set_tags(temp, filename, kwargs):
     """
      sets song tags after they are downloaded
-    :param temp: contains index used to obtain more info about song being editted
+    :param temp: contains index used to obtain more info about song being edited
-    :param filename: location of song whose tags are to be editted
+    :param filename: location of song whose tags are to be edited
     :param kwargs: a dictionary of extra arguments to be used in tag editing
     """
     song = kwargs["track_db"][int(temp[-1])]

utils/cut_large_file.py

@@ -1,9 +1,30 @@
 # trunk-ignore-all(pylint/C0301)
+"""
+The function `cut_my_life_into_pieces` reads a source file, splits its content based on a file size
+limit, and writes the parts to separate output files in a specified folder.
 
-FILE_SIZE_LIMIT = 100000 * 1024
+:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
-SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
+content that you want to write to a file. It represents the actual content that will be written to
-OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
+the file
-OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
+:param file: The code you provided defines two functions: `smart_write` and
+`cut_my_life_into_pieces`. The `smart_write` function encodes input lines to UTF-8, writes them to a
+file, and returns the size of the encoded bytes. The `cut_my_life_into_pieces` function reads
+:return: The functions provided in the code snippet do not explicitly return any values. However,
+the `smart_write` function returns the size of the encoded bytes of the input `lines` in bytes. The
+`cut_my_life_into_pieces` function does not have a return statement, so it does not explicitly
+return any value.
+"""
+
+FILE_SIZE_LIMIT = 50000 * 1024
+
+
+SOURCE = r"C:\Database\dois-2022-02-12\sci-hub-doi-2022-02-12.txt"
+OUTPUT_FOLDER = r"C:\Database\\chunks\\"
+OUTPUT_FILE_SCHEMA = r"_chunk.txt"
+
+#SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
+#OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
+#OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
 
 
 def smart_write(lines, file):
@@ -18,9 +39,12 @@ def smart_write(lines, file):
     :return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
     bytes.
     """
-    encoded_bytes = lines.encode("utf-8")
+    try:
-    size_in_bytes = len(encoded_bytes)
+        encoded_bytes = lines.encode("utf-8")
-    file.write(lines)
+        size_in_bytes = len(encoded_bytes)
+        file.write(lines)
+    except:
+        return 0
 
     return size_in_bytes
 
@@ -44,8 +68,9 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
     with open(source, encoding="utf-8") as infile:
         output_size = 0
         output_no = 0
-
+        lineno = 0
         for line in infile:
+            lineno += 1
             if output_size > FILE_SIZE_LIMIT:
                 output_no += 1
                 output_size = 0
@@ -55,4 +80,12 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
                 "a",
                 encoding="utf-8",
             ) as op_file:
-                output_size += smart_write(line, op_file)
+                tmp =  smart_write(line, op_file)
+                if tmp > 0:
+                    output_size += smart_write(line, op_file)
+                else:
+                    print (f'Error at {lineno}')
+                    print(line)
+
+if __name__ == "__main__":
+    cut_my_life_into_pieces(SOURCE,OUTPUT_FOLDER,OUTPUT_FILE_SCHEMA)