Tag: 1.0

Intermediate commits (oldest → newest):
- New files for flask services
- Service
- Minor fix 2
- Waitress
- Linux conf files
- Service
- Bugfixing
- Waitress
- Install options
- Work on deployment
- Bugfixes
- Work on deployment
- name change and othre bugfixes
- Installer
- Test

bot.py

@@ -706,6 +706,8 @@ async def get_file(ctx, source, link):
                     "force_keyframes": True,
                 },
             ]
+            #TODO: Make sponsorblock work
+            sponsorblock_postprocessor = []
             if platform == "win32":
                 dir_path = "G:\\Muzyka\\Youtube"
             else:
@@ -714,7 +716,6 @@ async def get_file(ctx, source, link):
                 "proxy": "",
                 "default_search": "ytsearch",
                 "format": "bestaudio/best",
-                "postprocessors": sponsorblock_postprocessor,
                 "noplaylist": True,
                 "no_color": False,
                 "paths": {"home": dir_path},
@@ -824,7 +825,7 @@ async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None):
     It is an optional parameter with a default value of 0, which means that if no value is provided for
     how_many, the function will return all search results, defaults to 0 (optional)
     """
-    # TODO: Potem dorobić wyszukiwania po kawałkach słów
+    #TODO: Przerobić na webservice
     # i sieć neuronową z możliwością wyrażenia opinii o dopasowaniu.
     if slowa_kluczowe:
         word_list = slowa_kluczowe
@@ -1299,7 +1300,8 @@ async def zagraj_mi_kawalek(ctx):
     async with ctx.typing():
         search_time_glob = datetime.now()
         logger.info("Zaczynam szukać timestamp %s", datetime.now())
-        file = await wyszukaj(ctx=ctx)
+        coro = asyncio.to_thread(wyszukaj,ctx=ctx)
+        file = await coro
         logger.info(
             "Koniec szukania(timestamp %s zajęło %s",
             datetime.now(),

conjurer.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/bot.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

conjurer_librarian/conjurer_librarian.py

@@ -0,0 +1,190 @@
+import re
+import logging
+import threading
+import time
+from flask import Flask, jsonify, request
+import netrc
+import json
+from json.decoder import JSONDecodeError
+from platform import uname
+from sys import platform
+from urllib.request import urlopen
+from habanero import Crossref
+from waitress import serve
+
+if platform in ("linux", "linux2"):
+    SEPARATOR_FILE_PATH = "/"
+    if "microsoft-standard" in uname().release:
+        NETRC_FILE = "/home/mtuszowski/.netrc"
+    else:
+        NETRC_FILE = "/home/pi/.netrc"
+elif platform == "win32":
+    NETRC_FILE = "C:\\Users\\mtusz\\.netrc"
+
+#HOST_ADDRESS = "192.168.1.191"
+HOST_ADDRESS = "0.0.0.0"
+PORT_ADDRESS = 5001
+
+class Librarian(object):
+
+    def __init__(self) -> None:
+        netrc_mod = netrc.netrc(NETRC_FILE)
+        authTokens = netrc_mod.authenticators("crossref")
+        self.cr = Crossref(mailto=authTokens[0])
+        self.limit = 1000
+        self.fetched = 0
+        self.hit = 0
+        self.total = 0
+
+    def search_crossref(self, query, uuid):
+        search_result_from_cr = self.cr.works(query = query, limit = self.limit)
+        self.total = search_result_from_cr['message']['total-results']
+        self.fetched = len(search_result_from_cr['message']['items'])
+        #add to sql with UUID
+        #sql row - title, doi, author, uuid, last retrieved, is_available
+        with open("results.json", "r+", encoding="UTF-8") as data_file:
+            # First we load existing data into a dict.
+            try:
+                file_data = json.load(data_file)
+            except JSONDecodeError:
+                file_data = []
+            # Join new_data with file_data inside emp_details
+            summarized_results = []
+            for item in search_result_from_cr['message']['items']:
+                container = "NO"
+                if "container-title" in item:
+                    container = "YES"
+                summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
+            result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'],  }
+            file_data.append(result)
+            print("Saving")
+            data_file.seek(0)
+            # convert back to json.
+            json.dump(file_data, data_file, indent=4)
+            return result
+
+
+    def check_if_exists(self,page_url):
+        #TODO: Spowolnic
+        #TODO: Sprawdzac po mirrorach
+        #TODO: sprawdzac w tym jebanym pliku
+        in_scihub_db = True
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        # trunk-ignore(bandit/B310)
+        with urlopen(page_url) as response:
+            data = response.read()
+            text = data.decode("utf-8")
+            for line in text.splitlines():
+                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                    return None
+        if in_scihub_db:
+            return text
+        return None
+
+    def download(self, page_url):
+        if response := self.check_if_exists(page_url):
+            for line in response.splitlines():
+                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
+                    return "https:" + str(m.group(1))
+        return None
+
+    def provide_sci_hub_link(self, page_url):
+        if self.check_if_exists(page_url):
+            return page_url
+        return None
+
+    def refine_query(self, unrefined_reqult):
+        print("Refine result")
+        refined_result = []
+        for item in unrefined_reqult["summary"]:
+            if item["container"] == "NO" and item["title"]:
+                item_link = "https://sci-hub.se/" + str(item['DOI'])
+                if self.check_if_exists(item_link):
+                    refined_result.append(item)
+                    print ("HIT")
+                    print(item["title"])
+                time.sleep(0.01)
+        print("Existing hits")
+        self.hit = len(refined_result)
+        return refined_result
+
+    def answer_query(self, query, uuid):
+        #Search sql for UUID
+        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        print("Answer result")
+        uuid_found = False
+        database = None
+        with open("results.json", "r", encoding="UTF-8") as new_file:
+            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
+            #ale tymczasowo plik
+            try:
+                database =  json.load(new_file)
+            except JSONDecodeError:
+                pass
+        if database:
+            for item in database:
+                if item["UUID"] == uuid:
+                    print (item["UUID"])
+                    uuid_found = True
+                    return self.refine_query(item)
+
+        if not uuid_found:
+            answer = self.search_crossref(query=query, uuid=uuid)
+            return self.refine_query(answer)
+
+#example not exists
+#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
+#sample usage to download
+#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
+#print(cl.download(sample_existing_page_url))
+#cl.search_crossref("BDSM")
+
+app = Flask(__name__)
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+@app.route("/query", methods=["POST"])
+def query_database():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["query"])
+    app.logger.info(record["UUID"])
+    cl = Librarian()
+    answer_data = cl.answer_query(record["query"], record["UUID"])
+    app.logger.info(answer_data)
+    summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
+    answer_data = (summary,answer_data)
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
+        200,
+    )
+    return return_data
+
+if __name__ == "__main__":
+    logger = logging.getLogger('conjurer_musician')
+    logger.setLevel(logging.DEBUG)
+    threads = []
+    #threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=flask_debug))
+
+    for worker in threads:
+        worker.start()
+    for worker in threads:
+        worker.join()
+
+#1. robi liste wyników i ją wyświetla - ale tylko tytuły niezależnych dzieł - jeśli jest coś w containerze to trza pogrupować
+#2. Tworzy przyciski pozwalające sprawdzić czy jest dostępne - i jeśli jest zwraca link
+#3. Jest też przycisk "more" - wyszukujemy po 100, wyświetlamy po 20. Po 100 jak jeszcze to dopiero odpytujemy crossref przyciski strzałek przesuwają po obecnej porcji danych

conjurer_librarian/install_librarian_service.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+python3 -m venv /home/pi/Conjurer_librarian/env
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+cp  /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
+cd /home/pi/Conjurer_librarian
+source /home/pi/Conjurer_librarian/env/bin/activate
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_librarian.txt
+deactivate
+
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/

conjurer_librarian/requirements_librarian.txt

@@ -0,0 +1,3 @@
+habanero
+waitress
+flask

deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+git pull
+cd ..
+cp ./conjurer/bot.py ./Conjurer/
+cp ./conjurer/settings.json ./Conjurer/
+cp ./conjurer/system_gpt_settings.json ./Conjurer/
+cp ./conjurer/install.sh ./Conjurer/
+cp ./conjurer/requirements.txt ./Conjurer/requirements.txt
+sudo systemctl restart conjurer.service

file_webservice/conjurer_musician.py

@@ -0,0 +1,134 @@
+"""
+The provided Python script sets up a Flask web server to manage a list of music files, with
+functions for rescanning the music folder, updating the music list, and serving the music list via
+API endpoints.
+"""
+import json
+from pathlib import Path
+from platform import uname
+import threading
+import time
+from sys import platform
+import logging
+from flask import Flask, jsonify, request
+from waitress import serve
+
+
+HOST_ADDRESS = "192.168.1.15"
+PORT_ADDRESS = 5000
+if platform in ("linux", "linux2"):
+    SEPARATOR_FILE_PATH = "/"
+    if "microsoft-standard" in uname().release:
+        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
+        MUSIC_FOLDER = "/mnt/g/Muzyka/"
+        NETRC_FILE = "/home/mtuszowski/.netrc"
+        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
+        ENCODING = "utf-8"
+        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
+
+    else:
+        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
+        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
+        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
+        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
+        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
+        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
+        NETRC_FILE = "/home/pi/.netrc"
+        LOGSTORE = "/home/pi/RetroPie/logs/"
+        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
+        ENCODING = "utf-8"
+        GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
+        DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
+
+
+music_file_list = []
+def rescan():
+    """
+    The `rescan` function logs a message, scans for mp3 files in a specified folder,
+    and adds them to a list of music files.
+    """
+    logging.info("Rescan triggered")
+    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+        temp_music_file = mp3_item.as_posix()
+        if platform == "win32":
+            temp_music_file = temp_music_file.replace("/", "\\")
+        music_file_list.append(temp_music_file)
+
+def thread_rescan():
+    """
+    The `thread_rescan` function periodically triggers a rescan operation after a specified time
+    interval.
+    """
+    logging.info("Starting filesystemupdater")
+    while True:
+        time.sleep(60*60*24)
+        logging.info("Rescan triggered")
+        rescan()
+
+app = Flask(__name__)
+
+@app.route("/mp3", methods=["GET"])
+def get_music_list():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
+    return jsonify({"music_file_list": music_file_list})
+
+
+@app.route("/update_mp3", methods=["POST"])
+def update_music_list():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record["item"])
+    music_file_list.append(record["item"])
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=record),
+        200,
+    )
+    return return_data
+
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def waitress_run():
+    serve(app, host="0.0.0.0", port=5000)
+
+if __name__ == "__main__":
+    logger = logging.getLogger('conjurer_musician')
+    logger.setLevel(logging.DEBUG)
+    rescan()
+    threads = []
+    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=thread_rescan))
+    #threads.append(threading.Thread(target=flask_debug))
+
+    for worker in threads:
+        worker.start()
+
+    for worker in threads:
+        worker.join()
+
+#TODO: wrzucić wyszukanie i robienie playlist
+#TODO: Inne pliki - sadox, dokumenty naukowe itp.

file_webservice/conjurer_musician.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer_musician
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

file_webservice/install_musician_service.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+python3 -m venv /home/pi/Conjurer-service/conjurer/file_webservice
+source ./bin/activate
+./bin/python3 -m pip install --upgrade pip
+./bin/python3 -m pip install -r requirements_musician.txt
+deactivate
+
+sudo cp ./conjurer_musician.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service

file_webservice/requirements_musician.txt

@@ -0,0 +1,2 @@
+flask
+waitress

install_main_bot.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+python3 -m venv /home/pi/Conjurer/env
+source ./env/bin/activate
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_bot.txt
+sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+deactivate
+sudo cp ./conjurer.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service

mp3_list.service.py

@@ -1,35 +0,0 @@
-from pathlib import Path, PurePath
-from platform import uname
-from sys import platform
-MUSIC_FOLDER = ""
-if platform in ("linux", "linux2"):
-    SEPARATOR_FILE_PATH = "/"
-    if "microsoft-standard" in uname().release:
-        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
-        MUSIC_FOLDER = "/mnt/g/Muzyka/"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
-
-    else:
-        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
-        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
-        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
-        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
-        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
-        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
-        NETRC_FILE = "/home/pi/.netrc"
-        LOGSTORE = "/home/pi/RetroPie/logs/"
-        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
-        DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
-
-music_file_list = []
-for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
-    temp_music_file = mp3_item.as_posix()
-    if platform == "win32":
-        temp_music_file = temp_music_file.replace("/", "\\")
-    music_file_list.append(temp_music_file)
-

requirements_bot.txt

@@ -0,0 +1,13 @@
+discord
+yt_dlp
+spotify_dl
+spotipy
+openai
+eyed3
+numpy
+pdf2image
+PyPDF2
+requests
+spotipy
+tiktoken
+PyNaCl

spotify_dl/youtube.py

@@ -97,8 +97,8 @@ def write_tracks(tracks_file, song_dict):
 def set_tags(temp, filename, kwargs):
     """
      sets song tags after they are downloaded
-    :param temp: contains index used to obtain more info about song being editted
-    :param filename: location of song whose tags are to be editted
+    :param temp: contains index used to obtain more info about song being edited
+    :param filename: location of song whose tags are to be edited
     :param kwargs: a dictionary of extra arguments to be used in tag editing
     """
     song = kwargs["track_db"][int(temp[-1])]

utils/cut_large_file.py

@@ -1,4 +1,19 @@
 # trunk-ignore-all(pylint/C0301)
+"""
+The function `cut_my_life_into_pieces` reads a source file, splits its content based on a file size
+limit, and writes the parts to separate output files in a specified folder.
+
+:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
+content that you want to write to a file. It represents the actual content that will be written to
+the file
+:param file: The code you provided defines two functions: `smart_write` and
+`cut_my_life_into_pieces`. The `smart_write` function encodes input lines to UTF-8, writes them to a
+file, and returns the size of the encoded bytes. The `cut_my_life_into_pieces` function reads
+:return: The functions provided in the code snippet do not explicitly return any values. However,
+the `smart_write` function returns the size of the encoded bytes of the input `lines` in bytes. The
+`cut_my_life_into_pieces` function does not have a return statement, so it does not explicitly
+return any value.
+"""
 
 FILE_SIZE_LIMIT = 100000 * 1024
 SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"