Bugfixes

asdasdasd

:)

asa

asdasda

aa

:)

asa

aa

:)

:)

sa

as

aa

aa

aa

aaa

as

11

aa

11

xxx

dd

ds

aa

a1

aa

Test

Suppress

Move

Installer

aaa

Serwis

`12`

aa

xd

asasa

xd

aa

asa

aaa

:)

Hejka

a

xd

aa

aaas

555

aaa

asa

aa

11

aaa

aaa

as

sa

async

asda

00

123

123

aa

as

ss

as

a

sa

aa

1

RES

T

Hejko

aa

11

AA

test

Test

conjurer_librarian/conjurer_librarian.py

@@ -0,0 +1,167 @@
+import re
+import logging
+import threading
+from flask import Flask, jsonify, request
+import netrc
+import json
+from json.decoder import JSONDecodeError
+from platform import uname
+from sys import platform
+from urllib.request import urlopen
+from habanero import Crossref
+from waitress import serve
+
+if platform in ("linux", "linux2"):
+    SEPARATOR_FILE_PATH = "/"
+    if "microsoft-standard" in uname().release:
+        NETRC_FILE = "/home/mtuszowski/.netrc"
+    else:
+        NETRC_FILE = "/home/pi/.netrc"
+elif platform == "win32":
+    NETRC_FILE = "C:\\Users\\mtusz\\.netrc"
+
+#HOST_ADDRESS = "192.168.1.191"
+HOST_ADDRESS = "0.0.0.0"
+PORT_ADDRESS = 5001
+
+class Librarian(object):
+
+    def __init__(self) -> None:
+        netrc_mod = netrc.netrc(NETRC_FILE)
+        authTokens = netrc_mod.authenticators("crossref")
+        self.cr = Crossref(mailto=authTokens[0])
+        self.last_search = ""
+        self.limit = 20
+
+    def search_crossref(self, query, uuid):
+        self.last_search = self.cr.works(query = query, limit = self.limit)
+        #add to sql with UUID
+        #sql row - title, doi, author, uuid, last retrieved, is_available
+        with open("results.json", "r+", encoding="UTF-8") as data_file:
+            # First we load existing data into a dict.
+            try:
+                file_data = json.load(data_file)
+            except JSONDecodeError:
+                file_data = []
+            # Join new_data with file_data inside emp_details
+            summarized_results = []
+            for item in self.last_search['message']['items']:
+                container = "NO"
+                if "container-title" in item:
+                    container = "YES"
+                print(item['title'])
+                print(item['DOI'])
+
+                summarized_results.append({"DOI" : item['DOI'], "title": item['title'], "type": item['type'], "container": container})
+            result = {"UUID": uuid, "total_results": self.last_search['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.last_search['message']['items'],  }
+            file_data.append(result)
+            print("Saving")
+            data_file.seek(0)
+            # convert back to json.
+            json.dump(file_data, data_file, indent=4)
+
+
+    def check_if_exists(self,page_url):
+        in_scihub_db = True
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        # trunk-ignore(bandit/B310)
+        with urlopen(page_url) as response:
+            info = response.info()
+            print(info.get_content_type())
+            data = response.read()
+            text = data.decode("utf-8")
+            for line in text.splitlines():
+                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                    return None
+        if in_scihub_db:
+            return text
+        return None
+
+    def download(self, page_url):
+        if response := self.check_if_exists(page_url):
+            for line in response.splitlines():
+                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
+                    return "https:" + str(m.group(1))
+        return None
+
+    def provide_sci_hub_link(self, page_url):
+        if self.check_if_exists(page_url):
+            return page_url
+        return None
+
+    def answer_query(self, query, uuid):
+        #Search sql for UUID
+        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        uuid_found = False
+        database = None
+        with open("results.json", "r", encoding="UTF-8") as new_file:
+            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
+            #ale tymczasowo plik
+            try:
+                database =  json.load(new_file)
+            except JSONDecodeError:
+                pass
+        if database:
+            for item in database:
+                if item["UUID"] == uuid:
+                    print (item["UUID"])
+                    uuid_found = True
+                    self.last_search = item
+            if uuid_found:
+                return self.last_search
+        self.search_crossref(query=query, uuid=uuid)
+        return self.last_search
+
+#example not exists
+#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
+#sample usage to download
+#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
+#print(cl.download(sample_existing_page_url))
+#cl.search_crossref("BDSM")
+
+app = Flask(__name__)
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+@app.route("/query", methods=["POST"])
+def query_database():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["query"])
+    app.logger.info(record["UUID"])
+    cl = Librarian()
+    answer_data = cl.answer_query(record["query"], record["UUID"])
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
+        200,
+    )
+    return return_data
+
+if __name__ == "__main__":
+    logger = logging.getLogger('conjurer_musician')
+    logger.setLevel(logging.DEBUG)
+    threads = []
+    #threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=flask_debug))
+
+    for worker in threads:
+        worker.start()
+    for worker in threads:
+        worker.join()
+
+#1. robi liste wyników i ją wyświetla - ale tylko tytuły niezależnych dzieł - jeśli jest coś w containerze to trza pogrupować
+#2. Tworzy przyciski pozwalające sprawdzić czy jest dostępne - i jeśli jest zwraca link
+#3. Jest też przycisk "more" - wyszukujemy po 100, wyświetlamy po 20. Po 100 jak jeszcze to dopiero odpytujemy crossref przyciski strzałek przesuwają po obecnej porcji danych

conjurer_librarian/install_librarian_service.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+python3 -m venv /home/pi/Conjurer_librarian/env
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+cp  /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
+cd /home/pi/Conjurer_librarian
+source /home/pi/Conjurer_librarian/env/bin/activate
+./env/bin/python3 -m pip install --upgrade pip
+./env/bin/python3 -m pip install -r requirements_librarian.txt
+deactivate
+
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/

conjurer_librarian/requirements_librarian.txt

@@ -0,0 +1,3 @@
+habanero
+waitress
+flask

deploy.sh

@@ -4,6 +4,6 @@ cd ..
 cp ./conjurer/bot.py ./Conjurer/
 cp ./conjurer/settings.json ./Conjurer/
 cp ./conjurer/system_gpt_settings.json ./Conjurer/
-cp ./conjurer/start.sh ./Conjurer/
+cp ./conjurer/install.sh ./Conjurer/
 cp ./conjurer/requirements.txt ./Conjurer/requirements.txt
 sudo systemctl restart conjurer.service

file_webservice/conjurer_musician.py

@@ -1,11 +1,21 @@
+"""
+The provided Python script sets up a Flask web server to manage a list of music files, with
+functions for rescanning the music folder, updating the music list, and serving the music list via
+API endpoints.
+"""
 import json
 from pathlib import Path
 from platform import uname
+import threading
+import time
 from sys import platform
-
+import logging
 from flask import Flask, jsonify, request
+from waitress import serve
 
-MUSIC_FOLDER = ""
+
+HOST_ADDRESS = "192.168.1.15"
+PORT_ADDRESS = 5000
 if platform in ("linux", "linux2"):
     SEPARATOR_FILE_PATH = "/"
     if "microsoft-standard" in uname().release:
@@ -30,24 +40,54 @@ if platform in ("linux", "linux2"):
         GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
         DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
 
+
 music_file_list = []
-for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+def rescan():
-    temp_music_file = mp3_item.as_posix()
+    """
-    if platform == "win32":
+    The `rescan` function logs a message, scans for mp3 files in a specified folder,
-        temp_music_file = temp_music_file.replace("/", "\\")
+    and adds them to a list of music files.
-    music_file_list.append(temp_music_file)
+    """
+    logging.info("Rescan triggered")
+    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+        temp_music_file = mp3_item.as_posix()
+        if platform == "win32":
+            temp_music_file = temp_music_file.replace("/", "\\")
+        music_file_list.append(temp_music_file)
+
+def thread_rescan():
+    """
+    The `thread_rescan` function periodically triggers a rescan operation after a specified time
+    interval.
+    """
+    logging.info("Starting filesystemupdater")
+    while True:
+        time.sleep(60*60*24)
+        logging.info("Rescan triggered")
+        rescan()
 
 app = Flask(__name__)
 
-
 @app.route("/mp3", methods=["GET"])
 def get_music_list():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
     return jsonify({"music_file_list": music_file_list})
 
 
-# TODO: Przyjecie informacji o zmianie i dodanie pliku saute
 @app.route("/update_mp3", methods=["POST"])
 def update_music_list():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
     record = json.loads(request.data)
     app.logger.info(record["item"])
     music_file_list.append(record["item"])
@@ -57,14 +97,34 @@ def update_music_list():
     )
     return return_data
 
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
 if __name__ == "__main__":
-    #app.run(debug=True, host="0.0.0.0")
+    logger = logging.getLogger('conjurer_musician')
-    from waitress import serve
+    logger.setLevel(logging.DEBUG)
-    serve(app, host="0.0.0.0", port=5000)
+    rescan()
+    threads = []
+    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=thread_rescan))
+    #threads.append(threading.Thread(target=flask_debug))
 
-# TODO: Reskan co 24h
+    for worker in threads:
+        worker.start()
 
-# TODO: Informacja o zmianie i update
+    for worker in threads:
+        worker.join()
 
-# TODO: Move download Youtube i Spotify tutaj (bedzie potrzebny wtedy rest w obie strony)
+#TODO: Inne pliki - sadox, dokumenty naukowe itp.

file_webservice/conjurer_musician.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer_musician
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

file_webservice/install_musician_service.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+python3 -m venv /home/pi/Conjurer-service/conjurer/file_webservice
+source ./bin/activate
+./bin/python3 -m pip install --upgrade pip
+./bin/python3 -m pip install -r requirements_musician.txt
+deactivate
+
+sudo cp ./conjurer_musician.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service

file_webservice/requirements_musician.txt

@@ -0,0 +1,2 @@
+flask
+waitress

install_main_bot.sh

@@ -2,8 +2,11 @@
 python3 -m venv /home/pi/Conjurer/env
 source ./env/bin/activate
 ./env/bin/python3 -m pip install --upgrade pip
-./env/bin/python3 -m pip install -r requirements.txt
+./env/bin/python3 -m pip install -r requirements_bot.txt
 sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
 sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
 deactivate
-sudo systemctl restart conjurer.service
+sudo cp ./conjurer.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service

requirements_bot.txt

@@ -9,4 +9,5 @@ pdf2image
 PyPDF2
 requests
 spotipy
-tiktoken
+tiktoken
+PyNaCl

spotify_dl/youtube.py

@@ -97,8 +97,8 @@ def write_tracks(tracks_file, song_dict):
 def set_tags(temp, filename, kwargs):
     """
      sets song tags after they are downloaded
-    :param temp: contains index used to obtain more info about song being editted
+    :param temp: contains index used to obtain more info about song being edited
-    :param filename: location of song whose tags are to be editted
+    :param filename: location of song whose tags are to be edited
     :param kwargs: a dictionary of extra arguments to be used in tag editing
     """
     song = kwargs["track_db"][int(temp[-1])]

utils/cut_large_file.py

@@ -1,4 +1,19 @@
 # trunk-ignore-all(pylint/C0301)
+"""
+The function `cut_my_life_into_pieces` reads a source file, splits its content based on a file size
+limit, and writes the parts to separate output files in a specified folder.
+
+:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
+content that you want to write to a file. It represents the actual content that will be written to
+the file
+:param file: The code you provided defines two functions: `smart_write` and
+`cut_my_life_into_pieces`. The `smart_write` function encodes input lines to UTF-8, writes them to a
+file, and returns the size of the encoded bytes. The `cut_my_life_into_pieces` function reads
+:return: The functions provided in the code snippet do not explicitly return any values. However,
+the `smart_write` function returns the size of the encoded bytes of the input `lines` in bytes. The
+`cut_my_life_into_pieces` function does not have a return statement, so it does not explicitly
+return any value.
+"""
 
 FILE_SIZE_LIMIT = 100000 * 1024
 SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"