Logging

jaki sposob inaczej przerobic fora na asynchroniczny. Przydatne ze
wzgledu na to ze szukajka nadal blokuje heartbeat bota - co skutkuje
warningnami i wyjatkami

communication_subroutine.py

@@ -1,76 +0,0 @@
-import logging
-import threading
-import json
-from queue import Queue, Empty
-from flask import Flask, jsonify, request
-from waitress import serve
-
-HOST_ADDRESS = "192.168.1.191"
-PORT_ADDRESS = 5000
-OUT_COMM_Q = Queue()
-IN_COMM_Q = Queue()
-internal_q = Queue()
-app = Flask(__name__)
-
-@app.route("/conjurer", methods=["POST"])
-def answer_external_command():
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info("DATA RECEIVED")
-
-    
-    while True:
-        try:
-            q_data = internal_q.get(block=False)
-        except Empty:
-            break
-    if q_data.uuid in record:
-        pass
-
-    return jsonify("SUCCESS")
-
-@app.route("/conjurer", methods=["GET"])
-def check_alive():
-    return jsonify("ALIVE")
-
-def flask_debug(_logger):
-    """
-    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
-    Do not use for production for fucks sake.
-    """
-    _logger.info("Attempt debug")
-    # trunk-ignore(bandit/B201)
-    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-
-def waitress_run(_logger):
-    """
-    The `waitress_run` function serves the `app` on host "0.0.0.0"
-    and port 5000 using the Waitress WSGI server.
-    """
-    _logger.info("Attempt waitress")
-    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-def scan_queue(_logger):
-    while True:
-        data = OUT_COMM_Q.get()
-        _logger.info(data)
-        internal_q.put(data)
-        query_list.append(data)
-
-def comm_subroutine(logger):
-    #logger.setLevel(logging.DEBUG)
-    logger.info("Started")
-    threads = []
-    threads.append(threading.Thread(target=flask_debug, args=(logger,)))
-    #threads.append(threading.Thread(target=waitress_run, args=(logger,)))
-    threads.append(threading.Thread(target=scan_queue,args=(logger,)))
-
-    for worker in threads:
-        worker.start()
-    for worker in threads:
-        worker.join()
-
-if __name__ == "__main__":
-    logging_client = logging.getLogger(__name__)
-    comm_subroutine(logging_client)

conjurer.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Conjurer
-After=multi-user.target
-
-[Service]
-Type=simple
-ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/bot.py
-Restart=on-abort
-
-[Install]
-WantedBy=multi-user.target

conjurer_librarian/conjurer_librarian.py

@@ -1,305 +0,0 @@
-import re
-import logging
-import threading
-import time
-from flask import Flask, jsonify, request
-import netrc
-import json
-import asyncio
-from json.decoder import JSONDecodeError
-from urllib.request import urlopen
-from habanero import Crossref
-from waitress import serve
-from queue import Queue
-import search_bot
-import requests
-from typing import Callable
-from uuid import UUID
-NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
-
-HOST_ADDRESS = "192.168.1.192"
-PORT_ADDRESS = 5001
-MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
-SEND_RESULTS = "/conjurer"
-BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
-
-ENCODING = "utf-8"
-app = Flask(__name__)
-
-librarian_queue = Queue()
-librarian_list = []
-
-class Librarian(object):
-
-    def __init__(self, app, query, uuid) -> None:
-        netrc_mod = netrc.netrc(NETRC_FILE)
-        authTokens = netrc_mod.authenticators("crossref")
-        self.cr = Crossref(mailto=authTokens[0])
-        self.query = query
-        self.uuid = str(uuid)
-        self.limit = 3000
-        self.fetched = 0
-        self.hit = 0
-        self.total = 0
-        self.app = app
-        self.logger = app.logger
-        self.live_results = []
-        self.search_result_from_cr = {}
-        self.done = False
-
-    async def search_crossref(self, query):
-        self.app.logger.info("STARTED SEARCH")
-        result = self.cr.works(query = query, limit = 1000)
-        self.search_result_from_cr.update(result)
-        self.total = result['message']['total-results']
-        self.fetched += len(result['message']['items'])
-        self.logger.info (self.total)
-        self.logger.info(self.fetched)
-        while self.total > self.fetched and self.limit > self.fetched:
-            result = self.cr.works(query = query, limit = 1000)
-            self.search_result_from_cr.update(result)
-            self.total = result['message']['total-results']
-            self.fetched += len(result['message']['items'])
-            self.logger.info (self.total)
-            self.logger.info(self.fetched)
-
-            time.sleep(1)
-        self.app.logger.info("CROSSREF DONE")
-        #TODO: add to sql with UUID
-        #sql row - title, doi, author, uuid, last retrieved, is_available
-        
-        with open("rr_results.json", "r+", encoding="utf-8") as data_file:
-            # First we load existing data into a dict.
-            try:
-                file_data = json.load(data_file)
-            except JSONDecodeError:
-                file_data = {}
-            summarized_results = []
-            self.app.logger.info("REFINE: Removing all derived works from the list")
-            for item in self.search_result_from_cr['message']['items']:
-                container = "NO"
-                if "container-title" in item:
-                    container = "YES"
-                else:
-                    summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
-            result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
-            if file_data:
-                file_data.update(result)
-            else:
-                file_data = result
-            
-            self.app.logger.info("REFINE: Dumping to file")
-            data_file.truncate(0)
-            data_file.seek(0)
-            json.dump(file_data, data_file, indent=4)
-            return result
-
-    async def check_dois_in_local_db(self, doi):
-        self.app.logger.info("REFINE: Running search in the backend app")
-        coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
-        result = await coro
-        return result
-
-    async def check_if_exists_brute_force(self, page_url):
-        self.app.logger.error("REFINE: Brute force search!")
-        if not page_url.startswith(("http:", "https:")):
-            raise ValueError("URL must start with 'http:' or 'https:'")
-        # trunk-ignore(bandit/B310)
-        with urlopen(page_url) as response:
-            data = response.read()
-            text = data.decode("utf-8")
-            for line in text.splitlines():
-                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
-                    return False
-                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
-                    direct_download_link = "https:" + str(m.group(1))
-                    self.app.logger.info (direct_download_link)
-                    return True
-        return False
-
-    async def check_if_exists(self, doi, brute_force = False):
-        result = {}
-        result = await self.check_dois_in_local_db(doi)
-        if brute_force and not result:
-            for item in doi:
-                item_link = "https://sci-hub.se/" + item[0]
-                tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
-                result.update(tmp)
-        return result
-
-    async def refine_query(self, unrefined_reqult, brute_force = False):
-        temp = []
-        refined_result = []
-        for key in unrefined_reqult:
-            self.logger.info("KEY:")
-            self.logger.info(key)
-
-        for item in unrefined_reqult[self.uuid]["summary"]:
-            if item["container"] == "NO" and item["title"]:                
-                refined_result.append(item)
-
-        for item in refined_result:
-            temp.append([item["DOI"], item])
-
-        refined_result = await self.check_if_exists(temp)
-        temp = []
-        for item in refined_result:
-            if item["exists"]:
-                temp.append(item["data"])
-        refined_result = temp
-        self.hit = len(refined_result)
-        return refined_result
-    #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
-    async def answer_query(self):
-        #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
-        #Search sql for UUID
-        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
-        self.logger.info(f"Search started {self.uuid}")
-        uuid_found = False
-        database = None
-        query = self.query
-        with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
-            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
-            #ale tymczasowo plik
-            try:
-                database =  json.load(cr_file)
-            except JSONDecodeError:
-                pass
-
-            if database:
-                for item in database.keys():
-                    if item in self.uuid:
-                        self.result = await self.refine_query(database)
-                        return self.result
-                answer = await self.search_crossref(query=query)
-                database.update(answer)
-                cr_file.truncate(0)
-                cr_file.seek(0)
-                json.dump(database,cr_file)
-                self.result = await self.refine_query(answer)
-                return self.result
-            else:
-                answer = await self.search_crossref(query=query)
-                database = answer
-                cr_file.truncate(0)
-                cr_file.seek(0)
-                json.dump(database,cr_file)
-                self.result = await self.refine_query(answer)
-                self.app.logger.info("Refined")
-                self.done = True
-                return self.result
-
-#============================= FLASK INTERNALS===============================
-def flask_debug():
-    """
-    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
-    Do not use for production for fucks sake.
-    """
-    # trunk-ignore(bandit/B201)
-    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-def waitress_run():
-    """
-    The `waitress_run` function serves the `app` on host "0.0.0.0"
-    and port 5000 using the Waitress WSGI server.
-    """
-    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-class BackgroundTaskSearch(threading.Thread):
-    def run(self):
-        loop = asyncio.new_event_loop()  # loop = asyncio.get_event_loop()
-        loop.run_until_complete(self._run())
-        loop.close()
-
-    async def _run(self):
-        while True:
-            database = None
-            librarian = librarian_queue.get()
-            self.app.logger.info("STARTED")
-            result = await librarian.answer_query()
-            result = {librarian.uuid: result}
-            self.app.logger.info("Saving to file")
-            with open("s_results.json", "r+", encoding="utf-8") as s_file:
-                try:
-                    database =  json.load(s_file)
-                except JSONDecodeError:
-                    pass
-                if database:
-                    database.extend(result)
-                else:
-                    database = result
-                self.app.logger.info("DUMPING DATA")
-                s_file.truncate(0)
-                s_file.seek(0)
-                json.dump(database, s_file)
-            self.app.logger.info("FINISHED")
-
-            self.app.logger.info(result)
-            coroutine = asyncio.to_thread(
-            requests.post,
-            f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
-            json=result,
-            timeout=360,
-    )       
-            self.app.logger.info("SENT")
-            _ = await coroutine
-            self.app.logger.info("SEND CONFIRMED")
-            time.sleep(1)
-            
-#==================================SERVER ROUTES==========================================
-@app.route("/query", methods=["POST"])
-async def query_database():
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info(record["query"])
-    app.logger.info(record["UUID"])
-    uuid = record["UUID"]
-    #TODO: TESTING ONLY======================
-    uuid = UUID('{12345678-1234-5678-1234-567812345678}')
-
-    cl = Librarian(app, record["query"], uuid)
-    librarian_queue.put(cl)
-    librarian_list.append(cl)
-    answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
-        200,
-    )
-    return return_data
-
-@app.route("/get_partial_result", methods=["POST"])
-async def get_partial():
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info(record["UUID"])
-    for lib in librarian_list:
-        if lib.uuid == record["UUID"]:
-            answer_data = lib.live_results
-            break
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
-        200,
-    )
-    return return_data
-
-#=======================================MAIN===================================================
-if __name__ == "__main__":
-    logger = logging.getLogger('conjurer_musician')
-    logger.setLevel(logging.DEBUG)
-    threads = []
-    #threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=flask_debug))
-    bgtask = BackgroundTaskSearch()
-    bgtask.app = app
-    threads.append(bgtask)
-    i = 0
-    for worker in threads:
-        try:
-            logger.info(f"App number: {i}")
-            i+=1
-            worker.start()
-        except RuntimeError as e:
-            logger.error("Exploded")
-            print(str(e))
-    for worker in threads:
-        worker.join()

conjurer_librarian/install_librarian_service.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-python3 -m venv /home/pi/Conjurer_librarian/env
-cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
-cp  /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
-cd /home/pi/Conjurer_librarian
-source /home/pi/Conjurer_librarian/env/bin/activate
-./env/bin/python3 -m pip install --upgrade pip
-./env/bin/python3 -m pip install -r requirements_librarian.txt
-deactivate
-
-cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/

conjurer_librarian/requirements_librarian.txt

@@ -1,3 +0,0 @@
-habanero
-waitress
-flask

conjurer_librarian/search_bot.py

@@ -1,106 +0,0 @@
-from queue import Queue, Empty
-from threading import Thread
-q = Queue()
-
-#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy
-#MAXTHREADS = 3
-MAXTHREADS = 1
-
-#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
-DATABASE_PATH = r'C:\\Database\\chunks\\'
-
-ENCODING = "utf-8"
-CHUNk = "_chunk.txt"
-_sentinel = object()
-result_list = []
-
-#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
-
-def producer(out_q, control_q, filename):
-    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        lines = operated_file.readlines()
-        for line in lines:
-            out_q.put(line)
-            try:
-                check = control_q.get(block=False)
-            except Empty:
-                check = False
-                pass
-            if check is _sentinel:
-                print("TERM signal received")
-                control_q.put(check)
-                break
-    print(f"Worker finished: {filename}")
-    out_q.put(_sentinel)
- 
-#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
-def consumer(in_q, control_q, doi, live_results, logger):
-    sentinels = 0
-    for item in doi:
-        result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
-    while True:
-        #if in_q.empty():
-        #    print("Queue empty!")
-        #if in_q.full():
-        #    print("Queue full!")
-        done_check = True
-        data = in_q.get()
-        if data is _sentinel:
-            logger.info("Worker finished")
-            sentinels += 1
-        else:
-            for item in result_list:
-                if item["DOI"] in data:
-                    logger.info(data)
-                    logger.info("HIT")
-                    item["exists"] = True
-                    live_results.append(item)
-                done_check = done_check and item["exists"]
-            if done_check:
-                control_q.put(_sentinel)
-        if sentinels >= MAXTHREADS:
-            logger.info("All workers finished")
-            break
-
-
-def search_for_doi(doi, live_results, logger):
-    threads = []
-    work_q = Queue(maxsize=200000)
-    control_q = Queue()
-    t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
-    logger.info("Consumer thread created")
-    threads.append(t_cons)
-    for i in range(0,MAXTHREADS):
-        #filename = "test" + str(i) + CHUNk
-        filename = "40" + CHUNk
-        #filename = str(i) + CHUNk
-        logger.info(f"Creating worker thread no: {i}")
-        threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
-    for worker in threads:
-        worker.start()
-    for worker in threads:
-        worker.join()
-    return result_list
-
-
-
-if __name__ == "__main__":
-    import logging
-    logger = logging.getLogger()
-    logger.setLevel("DEBUG")
-    h1 = logging.StreamHandler()
-    logger.addHandler(h1)
-    logger.info("TEST RUN")
-    live_result = []
-    logger.info(search_for_doi([
-        ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
-        ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
-        ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
-        ("10.2165/00128415-200309690-00017","DATA"),
-        ("10.5772/48313","DATA"),
-        ("10.15803/ijnc.7.2_419","DATA"),
-        ("10.1051/0004-6361/201321596e","DATA"),
-        ("10.2307/40835941","DATA"),
-        ], live_result, logger))
-    logger.info(live_result)

cut_scihub_database.py

@@ -1,30 +1,9 @@
 # trunk-ignore-all(pylint/C0301)
-"""
-The function `cut_my_life_into_pieces` reads a source file, splits its content based on a file size
-limit, and writes the parts to separate output files in a specified folder.
 
-:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
+FILE_SIZE_LIMIT = 100000 * 1024
-content that you want to write to a file. It represents the actual content that will be written to
+SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
-the file
+OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
-:param file: The code you provided defines two functions: `smart_write` and
+OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
-`cut_my_life_into_pieces`. The `smart_write` function encodes input lines to UTF-8, writes them to a
-file, and returns the size of the encoded bytes. The `cut_my_life_into_pieces` function reads
-:return: The functions provided in the code snippet do not explicitly return any values. However,
-the `smart_write` function returns the size of the encoded bytes of the input `lines` in bytes. The
-`cut_my_life_into_pieces` function does not have a return statement, so it does not explicitly
-return any value.
-"""
-
-FILE_SIZE_LIMIT = 50000 * 1024
-
-
-SOURCE = r"C:\Database\dois-2022-02-12\sci-hub-doi-2022-02-12.txt"
-OUTPUT_FOLDER = r"C:\Database\\chunks\\"
-OUTPUT_FILE_SCHEMA = r"_chunk.txt"
-
-#SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
-#OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
-#OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
 
 
 def smart_write(lines, file):
@@ -39,12 +18,9 @@ def smart_write(lines, file):
     :return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
     bytes.
     """
-    try:
+    encoded_bytes = lines.encode("utf-8")
-        encoded_bytes = lines.encode("utf-8")
+    size_in_bytes = len(encoded_bytes)
-        size_in_bytes = len(encoded_bytes)
+    file.write(lines)
-        file.write(lines)
-    except:
-        return 0
 
     return size_in_bytes
 
@@ -68,9 +44,8 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
     with open(source, encoding="utf-8") as infile:
         output_size = 0
         output_no = 0
-        lineno = 0
+
         for line in infile:
-            lineno += 1
             if output_size > FILE_SIZE_LIMIT:
                 output_no += 1
                 output_size = 0
@@ -80,12 +55,4 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
                 "a",
                 encoding="utf-8",
             ) as op_file:
-                tmp =  smart_write(line, op_file)
+                output_size += smart_write(line, op_file)
-                if tmp > 0:
-                    output_size += smart_write(line, op_file)
-                else:
-                    print (f'Error at {lineno}')
-                    print(line)
-
-if __name__ == "__main__":
-    cut_my_life_into_pieces(SOURCE,OUTPUT_FOLDER,OUTPUT_FILE_SCHEMA)

deploy.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-git pull
-cd ..
-cp ./conjurer/bot.py ./Conjurer/
-cp ./conjurer/settings.json ./Conjurer/
-cp ./conjurer/system_gpt_settings.json ./Conjurer/
-cp ./conjurer/install.sh ./Conjurer/
-cp ./conjurer/requirements.txt ./Conjurer/requirements.txt
-sudo systemctl restart conjurer.service

file_webservice/conjurer_musician.py

@@ -1,276 +0,0 @@
-"""
-The provided Python script sets up a Flask web server to manage a list of music files, with
-functions for rescanning the music folder, updating the music list, and serving the music list via
-API endpoints.
-"""
-import json
-import re
-from pathlib import Path
-from platform import uname
-import threading
-import time
-from datetime import datetime
-from sys import platform
-import logging
-from flask import Flask, jsonify, request
-from waitress import serve
-
-
-HOST_ADDRESS = "192.168.1.15"
-PORT_ADDRESS = 5000
-if platform in ("linux", "linux2"):
-    SEPARATOR_FILE_PATH = "/"
-    if "microsoft-standard" in uname().release:
-        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
-        MUSIC_FOLDER = "/mnt/g/Muzyka/"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
-
-    else:
-        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
-        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
-        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
-        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
-        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
-        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
-        NETRC_FILE = "/home/pi/.netrc"
-        LOGSTORE = "/home/pi/RetroPie/logs/"
-        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
-        DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
-
-
-music_file_list = []
-def rescan():
-    """
-    The `rescan` function logs a message, scans for mp3 files in a specified folder,
-    and adds them to a list of music files.
-    """
-    logging.info("Rescan triggered")
-    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
-        temp_music_file = mp3_item.as_posix()
-        if platform == "win32":
-            temp_music_file = temp_music_file.replace("/", "\\")
-        music_file_list.append(temp_music_file)
-
-def thread_rescan():
-    """
-    The `thread_rescan` function periodically triggers a rescan operation after a specified time
-    interval.
-    """
-    logging.info("Starting filesystemupdater")
-    while True:
-        time.sleep(60*60*24)
-        logging.info("Rescan triggered")
-        rescan()
-
-app = Flask(__name__)
-
-def wyszukaj(word_list, how_many):
-    logger = logging.getLogger()
-    return_list = []
-    search_weight = []
-
-    for _ in range(len(music_file_list)):
-        search_weight.append((0, ""))
-
-    time_start = datetime.now()
-    if int(how_many) > 0:
-        skip_start = 2
-    else:
-        skip_start = 1
-    for word in word_list[skip_start:]:
-        token_weight = len(word)
-        logger.info("Słowo kluczowe: %s", word)
-        itr = 0
-        for file in music_file_list:
-            file = file.split(SEPARATOR_FILE_PATH)
-
-            char_remove = [
-                ".",
-                "^",
-                "$",
-                "*",
-                "+",
-                "?",
-                "{",
-                "}",
-                "[",
-                "]",
-                "\\",
-                "/",
-                "|",
-                "(",
-                ")",
-                "!",
-                ",",
-                "-",
-                ":",
-                "mp3",
-            ]
-            all_words = []
-            for f_iter in file:
-                for char in char_remove:
-                    f_iter = remove_characters(f_iter, char)
-                tmp = f_iter.split()
-                all_words.extend(tmp)
-            pingu = 1
-            pattern_len = len(all_words)
-            if platform == "win32":
-                skip = 2
-            else:
-                skip = 4
-            matched_times = 1
-            for itm in all_words[skip:]:
-                pingu += 1
-                pattern = ".*" + word + ".*"
-                if re.match(pattern, itm, re.IGNORECASE):
-                    temp_weight = (
-                        search_weight[itr][0]
-                        + (token_weight + (pingu**1.5) / pattern_len) / matched_times
-                    )
-                    search_weight[itr] = (
-                        temp_weight,
-                        music_file_list[itr],
-                    )
-                    matched_times += 1
-            itr += 1
-
-    logger.info("Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start)
-    time_start = datetime.now()
-    item_to_search = max_weight(search_weight)
-    itr = 0
-    if item_to_search == 0:
-        return None
-    not_found = True
-    return_list = []
-    if int(how_many) <= 0:
-        logger.info("Jeden plik do zagrania")
-        while not_found:
-            if search_weight[itr][0] == item_to_search:
-                return_list.append(search_weight[itr])
-                return return_list
-            itr += 1
-    else:
-        logger.info("Wiele plików do zagrania")
-        search_weight.sort(key=lambda x: x[0], reverse=True)
-        return_list.extend(search_weight[: int(how_many)])
-        return return_list
-
-def max_weight(lista):
-    """
-    Take a list of weights and return the maximum weight.
-
-    :param lista: It seems like the parameter `lista` is a list of items, possibly representing weights.
-    The function name `max_weight` suggests that the function is intended to find the maximum weight
-    from the list. However, without more context or information about the problem, it's difficult to say
-    for sure what the
-    """
-    maximum_weight = 0
-    for iterator in lista:
-        if iterator[0] > maximum_weight:
-            maximum_weight = iterator[0]
-    return maximum_weight
-
-
-def remove_characters(string, character):
-    """
-    The `remove_characters` function removes all occurrences of a specified character from a given
-    string.
-
-    :param string: The string parameter is the input string from which characters will be removed
-    :param character: The character parameter is the character that you want to remove from the string
-    :return: a new string where all occurrences of the specified character have been removed.
-    """
-    return string.replace(character, "")
-
-
-@app.route("/mp3", methods=["GET"])
-def get_music_list():
-    """
-    The function `get_music_list` returns a JSON object containing a list of music files.
-    :return: A JSON response containing a key "music_file_list" with the value of the variable
-    `music_file_list`.
-    """
-    return jsonify({"music_file_list": music_file_list})
-
-
-@app.route("/update_mp3", methods=["POST"])
-def update_music_list():
-    """
-    The function `update_music_list` receives a POST request with a JSON payload, logs the received
-    item, adds it to a music file list, and returns a success message along with the updated record.
-    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
-    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
-    with values indicating the success of the operation and the data that was
-    received and added to the `music_file_list`.
-    The status code returned is 200, indicating a successful response.
-    """
-    record = json.loads(request.data)
-    app.logger.info(record["item"])
-    music_file_list.append(record["item"])
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=record),
-        200,
-    )
-    return return_data
-
-@app.route("/get_music", methods=["POST"])
-def look_for_playlist():
-    """
-    The function `update_music_list` receives a POST request with a JSON payload, logs the received
-    item, adds it to a music file list, and returns a success message along with the updated record.
-    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
-    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
-    with values indicating the success of the operation and the data that was
-    received and added to the `music_file_list`.
-    The status code returned is 200, indicating a successful response.
-    """
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info(record["lista_slow"])
-    app.logger.info(record["UUID"])
-    app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(record["lista_slow"],record["dlugosc_plejlisty"])
-
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
-        200,
-    )
-    return return_data
-
-
-def flask_debug():
-    """
-    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
-    Do not use for production for fucks sake.
-    """
-    # trunk-ignore(bandit/B201)
-    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-def waitress_run():
-    """
-    The `waitress_run` function serves the `app` on host "0.0.0.0"
-    and port 5000 using the Waitress WSGI server.
-    """
-    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-if __name__ == "__main__":
-    logger = logging.getLogger('conjurer_musician')
-    logger.setLevel(logging.DEBUG)
-    rescan()
-    logger.info("Started")
-    threads = []
-    threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=thread_rescan))
-
-    for worker in threads:
-        worker.start()
-
-    for worker in threads:
-        worker.join()
-
-#TODO: Inne pliki - sadox, dokumenty naukowe itp.

file_webservice/conjurer_musician.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Conjurer_musician
-After=multi-user.target
-
-[Service]
-Type=simple
-ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
-Restart=on-abort
-
-[Install]
-WantedBy=multi-user.target

file_webservice/install_musician_service.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-python3 -m venv /home/pi/Conjurer-service/conjurer/file_webservice
-source ./bin/activate
-./bin/python3 -m pip install --upgrade pip
-./bin/python3 -m pip install -r requirements_musician.txt
-deactivate
-
-sudo cp ./conjurer_musician.service /etc/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start conjurer_musician.service
-sudo systemctl enable conjurer_musician.service

file_webservice/requirements_musician.txt

@@ -1,2 +0,0 @@
-flask
-waitress

install_main_bot.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-python3 -m venv /home/pi/Conjurer/env
-source ./env/bin/activate
-./env/bin/python3 -m pip install --upgrade pip
-./env/bin/python3 -m pip install -r requirements_bot.txt
-sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
-sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
-deactivate
-sudo cp ./conjurer.service /etc/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start conjurer_musician.service
-sudo systemctl enable conjurer_musician.service

mp3_list.service.py

@@ -0,0 +1,35 @@
+from pathlib import Path, PurePath
+from platform import uname
+from sys import platform
+MUSIC_FOLDER = ""
+if platform in ("linux", "linux2"):
+    SEPARATOR_FILE_PATH = "/"
+    if "microsoft-standard" in uname().release:
+        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
+        MUSIC_FOLDER = "/mnt/g/Muzyka/"
+        NETRC_FILE = "/home/mtuszowski/.netrc"
+        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
+        ENCODING = "utf-8"
+        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
+
+    else:
+        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
+        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
+        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
+        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
+        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
+        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
+        NETRC_FILE = "/home/pi/.netrc"
+        LOGSTORE = "/home/pi/RetroPie/logs/"
+        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
+        ENCODING = "utf-8"
+        GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
+        DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
+
+music_file_list = []
+for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+    temp_music_file = mp3_item.as_posix()
+    if platform == "win32":
+        temp_music_file = temp_music_file.replace("/", "\\")
+    music_file_list.append(temp_music_file)
+

requirements_bot.txt

@@ -1,15 +0,0 @@
-discord
-yt_dlp
-spotify_dl
-spotipy
-openai
-eyed3
-numpy
-pdf2image
-PyPDF2
-requests
-spotipy
-tiktoken
-PyNaCl
-flask
-waitress

s_results.json

@@ -1 +0,0 @@
-{"12345678-1234-5678-1234-567812345678": [{"DOI": "10.1057/9781137435026", "title": ["Queer BDSM Intimacies"], "type": "book", "container": "NO"}]}

spotify_dl/youtube.py

@@ -97,8 +97,8 @@ def write_tracks(tracks_file, song_dict):
 def set_tags(temp, filename, kwargs):
     """
      sets song tags after they are downloaded
-    :param temp: contains index used to obtain more info about song being edited
+    :param temp: contains index used to obtain more info about song being editted
-    :param filename: location of song whose tags are to be edited
+    :param filename: location of song whose tags are to be editted
     :param kwargs: a dictionary of extra arguments to be used in tag editing
     """
     song = kwargs["track_db"][int(temp[-1])]