Bugfix in script deploy_music.sh

Adding playlists, various bufixes and installation scripts

Fixing

Rename

KUUUURWA

Install script music bot

fx

Fix ?

fix

Synchro

test

test

Undo

tst

test

tst

Undo changes

test

fx

tst

fx

Test commit of two files priority queue.

.gitignore

@@ -197,7 +197,3 @@ node_modules/prettier/esm/parser-postcss.mjs
 node_modules/prettier/esm/parser-typescript.mjs
 node_modules/prettier/esm/parser-yaml.mjs
 node_modules/prettier/esm/standalone.mjs
-cr_results.json
-not_in_db.json
-rr_results.json
-s_results.json

.trunk/configs/.shellcheckrc

@@ -1,7 +0,0 @@
-enable=all
-source-path=SCRIPTDIR
-disable=SC2154
-
-# If you're having issues with shellcheck following source, disable the errors via:
-# disable=SC1090
-# disable=SC1091

.trunk/trunk.yaml

@@ -2,38 +2,34 @@
 # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-  version: 1.22.2
+  version: 1.20.1
 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
     - id: trunk
-      ref: v1.6.0
+      ref: v1.4.5
       uri: https://github.com/trunk-io/plugins
 # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
 runtimes:
   enabled:
-    - go@1.21.0
     - node@18.12.1
     - python@3.10.8
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   enabled:
-    - shellcheck@0.10.0
-    - shfmt@3.6.0
-    - taplo@0.8.1
-    - actionlint@1.7.1
-    - bandit@1.7.9
-    - black@24.4.2
+    - actionlint@1.6.27
+    - bandit@1.7.8
+    - black@24.3.0
     - codespell
-    - checkov@3.2.139
+    - checkov@3.2.48
     - git-diff-check
     - isort@5.13.2
     - pylint
-    - markdownlint@0.41.0
-    - prettier@3.3.2
-    - ruff@0.4.9
-    - trivy@0.52.2
-    - trufflehog@3.78.1
+    - markdownlint@0.39.0
+    - prettier@3.2.5
+    - ruff@0.3.4
+    - trivy@0.50.1
+    - trufflehog@3.71.0
     - yamllint@1.35.1
 actions:
   disabled:

cogs/music.py

@@ -1,46 +0,0 @@
-import discord
-from discord.ext import commands
-from typing import  Optional
-
-class Music(commands.Cog):
-    def __init__(self, bot):
-        self.bot = bot # This is so you can access Bot instance in your cog
-
-# You must have this function for `bot.load_extension` to call
-def setup(bot):
-    bot.add_cog(Music(bot))
-
-@commands.hybrid_command(
-    name="przytul", description="Przytul kogoś - daj mention po komendzie :)"
-)
-async def przytul(ctx, arg: Optional[discord.Member] = None):
-    """
-    Generate a text about hugging mentioned user.
-
-    :param ctx: ctx stands for "context" and is a required parameter in Discord.py commands. It
-    represents the context in which the command was invoked, including information such as the message,
-    the channel, the server, and the user who invoked the command
-    :param arg: arg is a parameter of the function "przytul" that expects a Discord member object. The
-    parameter is optional, meaning that if no member object is provided, it will default to None
-    :type arg: Optional[discord.Member]
-    """
-    async with ctx.typing():
-        nieprzytulac = False
-        for mention in ctx.message.mentions:
-            for role in mention.roles:
-                if role.name == "NIEPRZYTULAĆ!":
-                    nieprzytulac = True
-
-        if arg and nieprzytulac:
-            await ctx.send(
-                f"Żebym ja Ciebie nie przytulił {ctx.message.author.mention}"
-            )
-        elif arg:
-            await ctx.send(
-                # trunk-ignore(codespell/misspelled)
-                f"Już dobrze.... Już dobrze... Ojej.. Biedactwo... :( *W ułamku sekundy {arg.mention} znajduje sie w duszącym uścisku. Żebra trzeszczą - kilka pęka. Pacnięcia po plecach grożą odbiciem nerek, a głaskanie po głowie powoduje wstrząs mózgu*"
-            )
-        else:
-            await ctx.send(
-                "Kogo mam przytulić? *Wyłamuje kostki i przeciąga się - jego 200 kilowa sylwetka złożona z samych mięśni świadczy o tym że jest gotowy*"
-            )

communication_subroutine.py

@@ -1,7 +1,6 @@
 import logging
 import threading
 import json
-import time
 from queue import Queue, Empty
 from flask import Flask, jsonify, request
 from waitress import serve
@@ -10,161 +9,62 @@ HOST_ADDRESS = "192.168.1.191"
 PORT_ADDRESS = 5000
 OUT_COMM_Q = Queue()
 IN_COMM_Q = Queue()
-awaiting_q = []
-incoming_q = Queue()
+internal_q = Queue()
 app = Flask(__name__)
-PREPPED_TRACKS = {
-    "requests" : "",
-    "hit" : "",
-    "all": "",
-    "priority":"",
-    "jingles":"",
-    "now_playing":""
-}
-logger = logging.getLogger("discord")
-
-class QueryControl:
-    """
-    This class `QueryControl` is used to manage queries with information about the author, UUID,
-    content, logger, context, and replies.
-    """
-    def __init__(self, query_author, query_uuid, query_content, ctx) -> None:
-        self.author = query_author
-        self.uuid = query_uuid
-        self.content = query_content
-        self.logger = logging.getLogger("discord")
-        self.stop = False
-        self.ctx = ctx
-        self.logger.info(
-            f"Created Query control for {self.author}, {self.uuid}: {self.content}"
-        )
-        self.replies = []
-
-@app.route("/prepped_tracks", methods=["POST"])
-def log_radio_tracks():
-    logger = logging.getLogger("discord")
-
-    logger.info(request)
-    record = json.loads(request.data)
-    logger.info(record)
-    PREPPED_TRACKS[record[0]] = record[1]
-    logger.info("DATA RECEIVED")
-    return jsonify("SUCCESS")
-
 
 @app.route("/conjurer", methods=["POST"])
 def answer_external_command():
-    """
-    The function `answer_external_command` logs the request data, loads the data as JSON, logs the
-    record, and then puts the record into an incoming queue before returning a success message.
-    :return: The function `answer_external_command()` is returning a JSON response with the message
-    "SUCCESS".
-    """
-    logger = logging.getLogger("discord")
-    logger.info(request)
     record = json.loads(request.data)
-    logger.info(record)
-    logger.info("DATA RECEIVED")
-    incoming_q.put(record)
+    app.logger.info(record)
+    app.logger.info("DATA RECEIVED")
+
+    
+    while True:
+        try:
+            q_data = internal_q.get(block=False)
+        except Empty:
+            break
+    if q_data.uuid in record:
+        pass
+
     return jsonify("SUCCESS")
 
 @app.route("/conjurer", methods=["GET"])
 def check_alive():
-    """
-    The function "check_alive" returns a JSON response with the message "ALIVE" when the "/conjurer"
-    route is accessed via a GET request.
-    :return: The code snippet is a Flask route that listens for GET requests to the "/conjurer"
-    endpoint. When a GET request is made to this endpoint, the function check_alive() is called, which
-    returns a JSON response with the message "ALIVE".
-    """
     return jsonify("ALIVE")
 
-def flask_debug():
+def flask_debug(_logger):
     """
     The `flask_debug` function starts a Flask application in debug mode without using the reloader.
     Do not use for production for fucks sake.
     """
-    logger = logging.getLogger("discord")
-
-    logger.info("Attempt debug")
+    _logger.info("Attempt debug")
     # trunk-ignore(bandit/B201)
     app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
 
-def waitress_run():
+def waitress_run(_logger):
     """
     The `waitress_run` function serves the `app` on host "0.0.0.0"
     and port 5000 using the Waitress WSGI server.
     """
-    logger = logging.getLogger("discord")
-
-    logger.info("Attempt waitress")
+    _logger.info("Attempt waitress")
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
-def scan_queue():
-    """
-    The function `scan_queue` reads data from a queue, logs it, and appends it to another queue.
-
-    :param _logger: The `_logger` parameter is typically an instance of a logging object that is used to
-    record and store log messages. It is commonly used to track the flow of the program, record errors,
-    and provide information for debugging purposes. In this code snippet, the `_logger` object is used
-    to log the
-    """
-    logger = logging.getLogger("discord")
+def scan_queue(_logger):
     while True:
         data = OUT_COMM_Q.get()
-        logger.info(data)
-        awaiting_q.append(data)
+        _logger.info(data)
+        internal_q.put(data)
+        query_list.append(data)
 
-def scan_incoming():
-    """
-    The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data
-    is found.
-
-    :param _logger: The `_logger` parameter in the `scan_incoming` function is a logger object that is
-    used to log messages or information during the execution of the function. It is typically used for
-    debugging, monitoring, or tracking the flow of the program. In this case, the `_logger` is being
-    used
-    """
-    logger = logging.getLogger("discord")
-    while True:
-        try:
-            answer = incoming_q.get(block=False)
-            logger.info("DATA FOUND")
-            record_stored = False
-            for record in awaiting_q:
-                if record.uuid in answer.keys():
-                    record_stored = True
-                    record.stop = True
-                    record.entries = answer[record.uuid]
-                    IN_COMM_Q.put(record)
-            if not record_stored:
-                for key in answer.keys():
-                    record = QueryControl("Orphaned", key, "Orphan", None)
-                    record.stop = True
-                    record.entries = answer[record.uuid]
-                    IN_COMM_Q.put(record)
-        except Empty:
-            time.sleep(1)
-
-
-def comm_subroutine():
-    """
-    The `comm_subroutine` function starts multiple threads to run different tasks concurrently.
-
-    :param logger: The `logger` parameter in the `comm_subroutine` function is an instance of a logger
-    object that is used for logging messages at various levels (e.g., debug, info, warning, error). In
-    the provided code snippet, the logger is used to log messages at the "info" level
-    """
+def comm_subroutine(logger):
     #logger.setLevel(logging.DEBUG)
-    logger = logging.getLogger("discord")
-    logger.info("Started comms")
+    logger.info("Started")
     threads = []
-    #threads.append(threading.Thread(target=flask_debug))
-    threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=scan_queue))
-    threads.append(threading.Thread(target=scan_incoming))
-
+    threads.append(threading.Thread(target=flask_debug, args=(logger,)))
+    #threads.append(threading.Thread(target=waitress_run, args=(logger,)))
+    threads.append(threading.Thread(target=scan_queue,args=(logger,)))
 
     for worker in threads:
         worker.start()
@@ -172,4 +72,5 @@ def comm_subroutine():
         worker.join()
 
 if __name__ == "__main__":
-    comm_subroutine()
+    logging_client = logging.getLogger(__name__)
+    comm_subroutine(logging_client)

conjurer_librarian/conjurer_librarian.py

@@ -1,316 +1,231 @@
-"""
-This module contains the implementation of the Librarian class and
-related functions for searching and refining queries.
-
-Classes:
--   Librarian: Represents a librarian object that performs search and
-    refinement operations on queries
-
-Functions:
--   flask_debug: Starts a Flask application in debug mode without using the reloader.
--   waitress_run: Serves the Flask application using the Waitress WSGI server.
--   BackgroundTaskSearch: Represents a background task for running the
-    Librarian object asynchronously
-"""
-
 import asyncio
 import json
 import logging
 import netrc
+import re
 import threading
 import time
 from json.decoder import JSONDecodeError
-from logging import handlers
 from queue import Queue
+from typing import Callable
+from urllib.request import urlopen
+from uuid import UUID
 
 import requests
-import scrape_bot
 import search_bot
-#import search_bot2 as search_bot
 from flask import Flask, jsonify, request
 from habanero import Crossref
 from waitress import serve
 
-# Constants
 NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
+
 HOST_ADDRESS = "192.168.1.192"
 PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
 BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
 
-MAX_CR_RESULTS = 1000
-#TEST PURPOSES ONLY!
-#MAX_CR_RESULTS = 5
-
 ENCODING = "utf-8"
-
 app = Flask(__name__)
 
 librarian_queue = Queue()
 librarian_list = []
 
 
-# trunk-ignore(pylint/R0902)
 class Librarian(object):
-    """
-    Represents a librarian object that performs search and refinement operations on queries.
-    """
 
-    def __init__(self, _app, query, uuid, _deep_search) -> None:
-        """
-        Initializes a Librarian object.
-
-        Args:
-        - _app: The Flask application object.
-        - query: The query to be searched.
-        - uuid: The unique identifier for the search.
-
-        Attributes:
-        - cr: The Crossref object for performing the search.
-        - query: The query to be searched.
-        - uuid: The unique identifier for the search.
-        - limit: The maximum number of search results to fetch.
-        - fetched: The number of search results fetched so far.
-        - hit: The number of search results that match the refinement criteria.
-        - total: The total number of search results.
-        - app: The Flask application object.
-        - live_results: A list to store live search results.
-        - final_result: A list to store the final refined search results.
-        - not_in_db: A list to store search results that are not in the local database.
-        - search_result_from_cr: A dictionary to store the search results from Crossref.
-        - done: A flag indicating if the search is done.
-        """
+    def __init__(self, app, query, uuid) -> None:
         netrc_mod = netrc.netrc(NETRC_FILE)
-        auth_tokens = netrc_mod.authenticators("crossref")
-        self.cr = Crossref(
-            mailto=auth_tokens[0],
-            ua_string=f"Conjurer project. mailto:{auth_tokens[0]}"
-        )
+        authTokens = netrc_mod.authenticators("crossref")
+        self.cr = Crossref(mailto=authTokens[0])
         self.query = query
         self.uuid = str(uuid)
-        self.limit = MAX_CR_RESULTS
+        self.limit = 3000
         self.fetched = 0
         self.hit = 0
         self.total = 0
-        self.app = _app
+        self.app = app
+        self.logger = app.logger
         self.live_results = []
-        self.final_result = {}
-        self.not_in_db = {}
         self.search_result_from_cr = {}
         self.done = False
-        self.deep_search = _deep_search
-
-    async def search_crossref(self, query, deep_search=False):
-        """
-        Performs a search on Crossref for the given query.
-
-        Args:
-        - query: The query to be searched.
-
-        Returns:
-        - result: The search result from Crossref.
-
-        Raises:
-        - None.
-        """
 
+    async def search_crossref(self, query):
         self.app.logger.info("STARTED SEARCH")
-        
-        if not deep_search:
-            query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000
-            cr_result = self.cr.works(query=query, limit=query_limit)
-            self.search_result_from_cr.update(cr_result)
-            self.total = cr_result["message"]["total-results"]
-            self.fetched += len(cr_result["message"]["items"])
-            self.app.logger.info(self.total)
-            self.app.logger.info(self.fetched)
-            while self.total > self.fetched and self.limit > self.fetched:
-                tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched)
-                cr_result["message"]["items"].extend(tmp_result["message"]["items"])
-                self.total = tmp_result["message"]["total-results"]
-                self.fetched = len(cr_result["message"]["items"])
-                self.app.logger.info(self.total)
-                self.app.logger.info(self.fetched)
-                time.sleep(0.1)
+        result = self.cr.works(query=query, limit=1000)
+        self.search_result_from_cr.update(result)
+        self.total = result["message"]["total-results"]
+        self.fetched += len(result["message"]["items"])
+        self.logger.info(self.total)
+        self.logger.info(self.fetched)
+        while self.total > self.fetched and self.limit > self.fetched:
+            result = self.cr.works(query=query, limit=1000)
+            self.search_result_from_cr.update(result)
+            self.total = result["message"]["total-results"]
+            self.fetched += len(result["message"]["items"])
+            self.logger.info(self.total)
+            self.logger.info(self.fetched)
 
-        else:
-            cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
-            result = cr_result[0]
-            for item in cr_result[1:]:
-                result["message"]["items"].extend(item["message"]["items"])
-                self.total = item["message"]["total-results"]
-                self.fetched = len(result["message"]["items"])
-                self.app.logger.info(self.total)
-                self.app.logger.info(self.fetched)
-            cr_result = result
-            self.app.logger.info("Total, fetched:")
-            self.app.logger.info(self.total)
-            self.app.logger.info(self.fetched)
-            self.search_result_from_cr.update(cr_result)
-            self.total = cr_result["message"]["total-results"]
-            self.app.logger.info("CROSSREF DONE")
+            time.sleep(1)
+        self.app.logger.info("CROSSREF DONE")
+        # TODO: add to sql with UUID
+        # sql row - title, doi, author, uuid, last retrieved, is_available
 
-            self.app.logger.info("CROSSREF DONE")
-        with open("cr_results.json", "r+", encoding="utf-8") as data_file:
-            # First we load existing data into a dict.
-            try:
-                file_data = json.load(data_file)
-            except JSONDecodeError:
-                file_data = {}
-            data_file.truncate(0)
-            data_file.seek(0)
-            tmp = {self.uuid : self.search_result_from_cr}
-            if file_data:
-                file_data.update(tmp)
-            else:
-                file_data = tmp
-            json.dump(file_data, data_file, indent=4)
-        return cr_result
-
-
-    async def refine_search(self, unrefined_result):
-        """
-        Refines the search query based on the unrefined search result.
-
-        Args:
-        - unrefined_result: The unrefined search result.
-
-        Returns:
-        - refined_result: The refined search result.
-
-        Raises:
-        - None.
-        """
-        summarized_results = []
-        self.app.logger.info("REFINE: Removing all derived works from the list")
-        for item in unrefined_result["message"]["items"]:
-            summarized_results.append(
-                {
-                    "DOI": item["DOI"],
-                    "title": item["title"] if "title" in item else None,
-                    "type": item["type"] if "type" in item else None,
-                }
-            )
-
-        partial_result = {
-            self.uuid: {
-                "total_results": self.search_result_from_cr["message"][
-                    "total-results"
-                ],
-                "on_page": 1,
-                "summary": summarized_results,
-                "results": self.search_result_from_cr["message"]["items"],
-            }
-        }
-        self.app.logger.info("REFINE: Dumping to file")
-        temp = []
-        refined_result = {}
-        for key in partial_result:
-            self.app.logger.info("KEY:")
-            self.app.logger.info(key)
-
-        for item in partial_result[self.uuid]["summary"]:
-            if item["title"]:
-                temp.append(item)
-
-        for item in temp:
-            refined_result[item["DOI"]]=  item
         with open("rr_results.json", "r+", encoding="utf-8") as data_file:
             # First we load existing data into a dict.
             try:
                 file_data = json.load(data_file)
             except JSONDecodeError:
                 file_data = {}
+            summarized_results = []
+            self.app.logger.info("REFINE: Removing all derived works from the list")
+            for item in self.search_result_from_cr["message"]["items"]:
+                container = "NO"
+                if "container-title" in item:
+                    container = "YES"
+                else:
+                    summarized_results.append(
+                        {
+                            "DOI": item["DOI"],
+                            "title": item["title"] if "title" in item else None,
+                            "type": item["type"] if "type" in item else None,
+                            "container": container,
+                        }
+                    )
+            result = {
+                self.uuid: {
+                    "total_results": self.search_result_from_cr["message"][
+                        "total-results"
+                    ],
+                    "on_page": 1,
+                    "summary": summarized_results,
+                    "results": self.search_result_from_cr["message"]["items"],
+                }
+            }
+            if file_data:
+                file_data.update(result)
+            else:
+                file_data = result
+
+            self.app.logger.info("REFINE: Dumping to file")
             data_file.truncate(0)
             data_file.seek(0)
-            tmp = {self.uuid: refined_result}
-            if file_data:
-                file_data.update(tmp)
-            else:
-                file_data = tmp
             json.dump(file_data, data_file, indent=4)
-        return refined_result
+            return result
 
-    async def check_if_exists(self, refined_result):
-        """
-        Checks if the given DOI exists.
-
-        Args:
-        - doi: The DOI to be checked.
-        - brute_force: A flag indicating if brute force method should be used.
-
-        Returns:
-        - result: The search result.
-
-        Raises:
-        - None.
-        """
-        result = {}
+    async def check_dois_in_local_db(self, doi):
         self.app.logger.info("REFINE: Running search in the backend app")
-        dois = []
-        for item, value in refined_result.items():
-            dois.append([item, value])
         coro = asyncio.to_thread(
-            search_bot.search_for_doi, dois, self.live_results, self.app.logger
+            search_bot.search_for_doi, doi, self.live_results, self.app.logger
         )
         result = await coro
-        result_list = []
-        result_no_db = []
-        for item in result:
+        return result
+
+    async def check_if_exists_brute_force(self, page_url):
+        self.app.logger.error("REFINE: Brute force search!")
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        # trunk-ignore(bandit/B310)
+        with urlopen(page_url) as response:
+            data = response.read()
+            text = data.decode("utf-8")
+            for line in text.splitlines():
+                if re.match(
+                    r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
+                    line,
+                ):
+                    return False
+                if m := re.match(
+                    r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
+                ):
+                    direct_download_link = "https:" + str(m.group(1))
+                    self.app.logger.info(direct_download_link)
+                    return True
+        return False
+
+    async def check_if_exists(self, doi, brute_force=False):
+        result = {}
+        result = await self.check_dois_in_local_db(doi)
+        if brute_force and not result:
+            for item in doi:
+                item_link = "https://sci-hub.se/" + item[0]
+                tmp = {
+                    "DOI": item["DOI"],
+                    "exists": await self.check_if_exists_brute_force(item_link),
+                    "data": item,
+                }
+                result.update(tmp)
+        return result
+
+    async def refine_query(self, unrefined_reqult, brute_force=False):
+        temp = []
+        refined_result = []
+        for key in unrefined_reqult:
+            self.logger.info("KEY:")
+            self.logger.info(key)
+
+        for item in unrefined_reqult[self.uuid]["summary"]:
+            if item["container"] == "NO" and item["title"]:
+                refined_result.append(item)
+
+        for item in refined_result:
+            temp.append([item["DOI"], item])
+
+        refined_result = await self.check_if_exists(temp)
+        temp = []
+        for item in refined_result:
             if item["exists"]:
-                result_list.append(item)
+                temp.append(item["data"])
+        refined_result = temp
+        self.hit = len(refined_result)
+        return refined_result
+
+    # TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
+    async def answer_query(self):
+        # TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
+        # Search sql for UUID
+        # SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        self.logger.info(f"Search started {self.uuid}")
+        uuid_found = False
+        database = None
+        query = self.query
+        with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
+            # tu logika komunikacji z SQL i cała logika związana z wyszukaniem
+            # ale tymczasowo plik
+            try:
+                database = json.load(cr_file)
+            except JSONDecodeError:
+                pass
+
+            if database:
+                for item in database.keys():
+                    if item in self.uuid:
+                        self.result = await self.refine_query(database)
+                        return self.result
+                answer = await self.search_crossref(query=query)
+                database.update(answer)
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database, cr_file)
+                self.result = await self.refine_query(answer)
+                return self.result
             else:
-                result_no_db.append(item)
-        self.hit = len(result)
-        return result_list, result_no_db
+                answer = await self.search_crossref(query=query)
+                database = answer
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database, cr_file)
+                self.result = await self.refine_query(answer)
+                self.app.logger.info("Refined")
+                self.done = True
+                return self.result
 
 
-    async def answer_query(self, deep_search=False):
-        """
-        Answers the search query.
-
-        Args:
-        - None.
-
-        Returns:
-        - result: The search result.
-
-        Raises:
-        - None.
-        """
-        self.app.logger.info(f"Search started {self.uuid}")
-        cr_result = await self.search_crossref(query=self.query, deep_search=deep_search)
-        refined_result = await self.refine_search(cr_result)
-        answer, negative_answer = await self.check_if_exists(refined_result)
-
-        self.app.logger.info("Returning result")
-        self.app.logger.info(answer)
-        self.app.logger.info(negative_answer)
-
-        for item in answer:
-            self.final_result[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
-        for item in negative_answer:
-            self.not_in_db[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
-        self.app.logger.info("Returning result case2")
-        self.app.logger.info(self.final_result)
-        return self.final_result
-
 # ============================= FLASK INTERNALS===============================
-
-
 def flask_debug():
     """
-    Starts a Flask application in debug mode without using the reloader.
-
-    Args:
-    - None.
-
-    Returns:
-    - None.
-
-    Raises:
-    - None.
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
     """
     # trunk-ignore(bandit/B201)
     app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
@@ -318,86 +233,33 @@ def flask_debug():
 
 def waitress_run():
     """
-    Serves the Flask application using the Waitress WSGI server.
-
-    Args:
-    - None.
-
-    Returns:
-    - None.
-
-    Raises:
-    - None.
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
 
 class BackgroundTaskSearch(threading.Thread):
-    """
-    A background task for searching and saving results to files.
-
-    This class extends the `threading.Thread` class and is responsible for running
-    the search task in the background. It retrieves queries from a queue, performs
-    the search, and saves the results to files.
-
-    Attributes:
-        app (App): The application instance.
-    """
-
     def run(self):
-        """
-        Run the background task.
-
-        This method is called when the thread is started. It creates a new event loop,
-        runs the `_run` method, and closes the event loop.
-        """
-        loop = asyncio.new_event_loop()
+        loop = asyncio.new_event_loop()  # loop = asyncio.get_event_loop()
         loop.run_until_complete(self._run())
         loop.close()
 
     async def _run(self):
-        """
-        Perform the search task.
-
-        This method is an asynchronous coroutine that runs in a loop. It retrieves a
-        librarian from the queue, answers the query, and saves the results to files.
-        It also sends the results to a remote server.
-
-        The search task continues running indefinitely until the thread is stopped.
-        """
         while True:
             database = None
-            ndb_database = None
             librarian = librarian_queue.get()
             self.app.logger.info("STARTED")
-            result = await librarian.answer_query(librarian.deep_search)
+            result = await librarian.answer_query()
             result = {librarian.uuid: result}
             self.app.logger.info("Saving to file")
-
-            # Save results to "not_in_db.json" file
-            with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
-                try:
-                    ndb_database = json.load(ndb_file)
-                except JSONDecodeError:
-                    pass
-                if ndb_database:
-                    ndb_database.update(librarian.not_in_db)
-                else:
-                    ndb_database = librarian.not_in_db
-                ndb_file.truncate(0)
-                ndb_file.seek(0)
-                json.dump(ndb_database, ndb_file)
-
-            # Save results to "s_results.json" file
             with open("s_results.json", "r+", encoding="utf-8") as s_file:
                 try:
                     database = json.load(s_file)
                 except JSONDecodeError:
                     pass
                 if database:
-                    self.app.logger.info(database)
-                    self.app.logger.info(result)
-                    database.update(result)
+                    database.extend(result)
                 else:
                     database = result
                 self.app.logger.info("DUMPING DATA")
@@ -414,8 +276,7 @@ class BackgroundTaskSearch(threading.Thread):
                 timeout=360,
             )
             self.app.logger.info("SENT")
-            result = await coroutine
-            self.app.logger.info(result.status_code)
+            _ = await coroutine
             self.app.logger.info("SEND CONFIRMED")
             time.sleep(1)
 
@@ -423,26 +284,15 @@ class BackgroundTaskSearch(threading.Thread):
 # ==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
-    """
-    Endpoint for querying the database.
-
-    This function receives a POST request containing a JSON payload with a query and a UUID.
-    It creates a Librarian object with the query and UUID,
-    and adds it to the librarian_queue and librarian_list.
-    Finally, it returns a JSON response indicating the success
-    of the operation, along with the query, UUID,
-    and the current size of the librarian_queue.
-
-    Returns:
-        tuple: A tuple containing a JSON response and a status code.
-    """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["query"])
     app.logger.info(record["UUID"])
     uuid = record["UUID"]
-    deep_search = record["deep_search"]
-    cl = Librarian(app, record["query"], uuid, deep_search)
+    # TODO: TESTING ONLY======================
+    uuid = UUID("{12345678-1234-5678-1234-567812345678}")
+
+    cl = Librarian(app, record["query"], uuid)
     librarian_queue.put(cl)
     librarian_list.append(cl)
     answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
@@ -455,12 +305,6 @@ async def query_database():
 
 @app.route("/get_partial_result", methods=["POST"])
 async def get_partial():
-    """
-    Retrieves the partial result for a given UUID.
-
-    Returns:
-        A JSON response containing the partial result.
-    """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["UUID"])
@@ -477,31 +321,22 @@ async def get_partial():
 
 # =======================================MAIN===================================================
 if __name__ == "__main__":
-    app.logger.setLevel(logging.DEBUG)
-    h1 = handlers.RotatingFileHandler(
-        filename="D:\\logs\\librarian.log",
-        encoding="utf-8",
-        mode="a",
-        maxBytes=6 * 1024 * 1024,
-        backupCount=6,
-    )
-
-    app.logger.addHandler(h1)
+    logger = logging.getLogger("conjurer_musician")
+    logger.setLevel(logging.DEBUG)
     threads = []
-    threads.append(threading.Thread(target=waitress_run))
-    # threads.append(threading.Thread(target=flask_debug))
+    # threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
     threads.append(bgtask)
-    threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
     i = 0
     for worker in threads:
         try:
-            app.logger.info("App number: %s", i)
+            logger.info(f"App number: {i}")
             i += 1
             worker.start()
         except RuntimeError as e:
-            app.logger.error("Exploded")
+            logger.error("Exploded")
             print(str(e))
     for worker in threads:
         worker.join()

conjurer_librarian/count_lines.py

@@ -1,8 +0,0 @@
-
-FILEPTH = r"C:\Database\chunks\4_chunk.txt"
-
-with open(FILEPTH, "r", encoding="utf-8") as file:
-    tab = file.readlines()
-    print(len(tab))
-
-    #1842295

conjurer_librarian/install_librarian_service.sh

@@ -1,33 +1,11 @@
 #!/bin/bash
-# This script sets up a virtual environment for the Conjurer Librarian service and installs the required dependencies.
-
-
-# Create a virtual environment at /home/pi/Conjurer_librarian/env
 python3 -m venv /home/pi/Conjurer_librarian/env
-
-# Copy the Conjurer Librarian Python script to the appropriate directory
-cp /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
-
-# Copy the requirements file to the appropriate directory
-cp /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
-
-# Change directory to /home/pi/Conjurer_librarian
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+cp  /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
 cd /home/pi/Conjurer_librarian
-
-touch /home/pi/Conjurer/rr_results.json
-touch /home/pi/Conjurer/cr_results.json
-touch /home/pi/Conjurer/s_results.json
-touch /home/pi/Conjurer/rr_results.json
-touch /home/pi/Conjurer/not_in_db.json
-
-# Activate the virtual environment
 source /home/pi/Conjurer_librarian/env/bin/activate
-
-# Upgrade pip
 ./env/bin/python3 -m pip install --upgrade pip
-
-# Install the required dependencies
 ./env/bin/python3 -m pip install -r requirements_librarian.txt
-
-# Deactivate the virtual environment
 deactivate
+
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/

conjurer_librarian/requirements_librarian.txt

@@ -1,3 +1,3 @@
 habanero
 waitress
-flask[async]
+flask

conjurer_librarian/run_server.bat

@@ -1 +0,0 @@
-c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py

conjurer_librarian/scrape_bot.py

@@ -1,136 +0,0 @@
-"""
-This module contains the code for the scrape bot.
-"""
-
-import json
-import logging
-import random
-import re
-import time
-from json import JSONDecodeError
-from queue import Queue
-from threading import Thread
-from urllib.request import urlopen
-
-from requests import ConnectionError as RequestsConnectionError
-from requests import ConnectTimeout, Timeout
-
-SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
-SCR_FILENAME = "40_chunk.txt"
-SCR_ENCODING = "utf-8"
-
-WORK_Q = Queue()
-random.seed()
-
-
-def load_ndb_to_q(logger):
-    """
-    This function loads items from the not_in_db.json file into the work queue.
-    """
-    logger.info("Loader started")
-    while True:
-        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
-            try:
-                ndb_database = json.load(ndb_file)
-                for _ in range (1,10):
-                    try:
-                        key = next(iter(ndb_database))
-                        _ = ndb_database.pop(key)
-                        logger.info(key)
-                        url = f"https://sci-hub.se/{key}"
-                        WORK_Q.put([key, url, False])
-                    except StopIteration:
-                        break
-                ndb_file.truncate(0)
-                ndb_file.seek(0)
-                json.dump(ndb_database, ndb_file, indent=4)
-            except JSONDecodeError:
-                time.sleep(60 * 60 * 3)
-        time.sleep(60*60*3)
-
-
-def check_if_exists_brute_force(logger):
-    """
-    This function checks if a document exists using brute force search.
-    """
-    # Function code here
-    while True:
-        logger.info("Scraper tick")
-        item = WORK_Q.get()
-        page_url = item[1]
-        logger.error("REFINE: Brute force search!")
-        if not page_url.startswith(("http:", "https:")):
-            raise ValueError("URL must start with 'http:' or 'https:'")
-        blocked = True
-        try:
-            # trunk-ignore(bandit/B310)
-            with urlopen(page_url) as response:
-                data = response.read()
-                text = data.decode("utf-8")
-                if item[2]:
-                    logger.info("Already found")
-                    time.sleep(180)
-                    blocked = False
-                else:
-                    for line in text.splitlines():
-                        if re.match(
-                            r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
-                            line,
-                        ):
-                            blocked = False
-                            logger.info("Not found")
-                            item[2] = False
-                        if m := re.match(
-                            r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
-                        ):
-                            blocked = False
-                            direct_download_link = "https:" + str(m.group(1))
-                            logger.info(direct_download_link)
-                            with open(
-                                SCR_DATABASE_PATH + SCR_FILENAME,
-                                "a",
-                                encoding=SCR_ENCODING,
-                            ) as operated_file:
-                                operated_file.write("\n")
-                                operated_file.write(item[0])
-                                item[2] = True
-        except (
-            Timeout,
-            ConnectTimeout,
-            ConnectionRefusedError,
-            ConnectionError,
-            RequestsConnectionError,
-        ):
-            pass
-        if blocked:
-            logger.info(item)
-            logger.info(text)
-            logger.error("Got blocked. Fuck.")
-            time.sleep(60 * 60 * 72)
-        # trunk-ignore(bandit/B311)
-        rand = random.randint(1, 60)
-        logger.info(f"Sleeping for {2*rand} minutes")
-        time.sleep(120 * rand)
-
-
-def scraper(logger=None):
-    """
-    This function is responsible for scraping data.
-    """
-    if not logger:
-        logger = logging.getLogger()
-        logger.setLevel("DEBUG")
-        h1 = logging.StreamHandler()
-        logger.addHandler(h1)
-        logger.info("No logger set. Test run")
-
-    loader = Thread(target=load_ndb_to_q, args=(logger,))
-    _scraper = Thread(target=check_if_exists_brute_force, args=(logger,))
-    loader.start()
-    _scraper.start()
-    loader.join()
-    _scraper.join()
-
-
-if __name__ == "__main__":
-    scraper()

conjurer_librarian/search_bot.py

@@ -1,170 +1,82 @@
-"""
-This module contains functions for searching for DOI (Digital Object Identifier)
-in a list of live results.
-It includes a producer-consumer pattern implementation using threads and queues.
-
-Functions:
-- producer: Reads lines from a file and puts them into an output queue.
-- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
-- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
-
-Global Variables:
-- MAXTHREADS: Maximum number of worker threads.
-- DATABASE_PATH: Path to the database files.
-- ENCODING: Encoding of the database files.
-- CHUNK: File name pattern for the database files.
-- _sentinel: Sentinel object used to signal termination.
-- result_list: List to store the search results.
-- WORK_Q_SIZE: Maximum size of the work queue.
-"""
-
-# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
-from queue import Empty, Queue
+from queue import Queue, Empty
 from threading import Thread
-import time
 q = Queue()
-#TODO: Count number of lines in files and print to approximate on which part of the file search is
 
-# DATA FOR TEST ONLY
-# MAXTHREADS = 5
-# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy
+#MAXTHREADS = 3
+MAXTHREADS = 1
 
-# DEPLOYMENT DATA
-MAXTHREADS = 41
-DATABASE_PATH = r"C:\\Database\\chunks\\"
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+DATABASE_PATH = r'C:\\Database\\chunks\\'
 
 ENCODING = "utf-8"
-CHUNK = "_chunk.txt"
+CHUNk = "_chunk.txt"
 _sentinel = object()
-WORK_Q_SIZE = 35500000
+result_list = []
 
+#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
 
-def producer(out_q, control_q, filename, _logger):
-    """
-    Produces items from the output queue and puts them into the control queue.
-
-    Args:
-        out_q (Queue): Output queue.
-        control_q (Queue): Control queue.
-        filename (str): Name of the file.
-        _logger: Logger object for logging.
-    """
+def producer(out_q, control_q, filename):
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        print(f"Worker {filename} ")
-        line_no = 0
-        while True:
-            line = operated_file.readline()
-            line_no += 1
-            print(f"\t \t \t \t \t \t W{filename}{line_no}\r", end="")
-
-            if not line:
-                print(f"EOF {filename}")
-                break
-
-            if not line:
-                break
+        lines = operated_file.readlines()
+        for line in lines:
             out_q.put(line)
             try:
                 check = control_q.get(block=False)
             except Empty:
                 check = False
-
+                pass
             if check is _sentinel:
                 print("TERM signal received")
                 control_q.put(check)
                 break
-        print(f"Worker finished: {filename}")
-        out_q.put(_sentinel)
-
-
-# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
-    """
-    Consumes items from an input queue and checks if DOI exists in the live results list.
-
-    Args:
-        in_q (Queue): Input queue.
-        control_q (Queue): Control queue.
-        doi (list): List of DOI to search for.
-        live_results (list): List to store the search results.
-        _logger: Logger object for logging.
-    """
-    print(f"Consumer thread started: {no} no")
-    empty_counter = 0
-    alive_no = 0
+    print(f"Worker finished: {filename}")
+    out_q.put(_sentinel)
+ 
+#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
+def consumer(in_q, control_q, doi, live_results, logger):
+    sentinels = 0
+    for item in doi:
+        result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
     while True:
+        #if in_q.empty():
+        #    print("Queue empty!")
+        #if in_q.full():
+        #    print("Queue full!")
         done_check = True
-        try:
-            data = in_q.get(block=True, timeout = 1)
-            if data is _sentinel:
-                control_dict["sentinels"] += 1
-                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
-
-            else:
-                empty_counter = 0
-                alive_no += 1
-                print(f"C{no}__{alive_no}\r", end="")
-
-                for item in result_list:
-                    if item["DOI"] in data and not item["exists"]:
-                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
-                        _logger.info(data)
-                        _logger.info("HIT")
-                        item["exists"] = True
-                        live_results.append(item)
-                    done_check = done_check and item["exists"]
-                if done_check:
-                    control_q.put(_sentinel)
-        except Empty:
-            empty_counter += 1
-            time.sleep(1)
-            print(f"Consumer {no} empty")
-            if empty_counter > 5:
-                print("Consumer %s empty lvl 2", no)
-                time.sleep(2)
-            elif empty_counter > 10:
-                print(f"Consumer thread finished {no}")
-                break
-
-        if control_dict["sentinels"] >= MAXTHREADS:
-            _logger.info(f"All workers finished {no}")
+        data = in_q.get()
+        if data is _sentinel:
+            logger.info("Worker finished")
+            sentinels += 1
+        else:
+            for item in result_list:
+                if item["DOI"] in data:
+                    logger.info(data)
+                    logger.info("HIT")
+                    item["exists"] = True
+                    live_results.append(item)
+                done_check = done_check and item["exists"]
+            if done_check:
+                control_q.put(_sentinel)
+        if sentinels >= MAXTHREADS:
+            logger.info("All workers finished")
             break
 
 
-
-def search_for_doi(doi, live_results, _logger):
-    """
-    Search for DOI in live_results using _logger for logging.
-
-    Args:
-        doi (list): List of DOI to search for.
-        live_results (list): List to store the search results.
-        _logger: Logger object for logging.
-    """
-    control_dict = {"sentinels":0}
-    result_list = []
+def search_for_doi(doi, live_results, logger):
     threads = []
-    work_q = Queue(maxsize=WORK_Q_SIZE)
+    work_q = Queue(maxsize=200000)
     control_q = Queue()
-
-    for item in doi:
-        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
-
-    for i in range (0, (len(doi)//1000)+2):
-        t_cons = Thread(
-            target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)
-        )
-        _logger.info("Consumer thread created")
-        threads.append(t_cons)
-    for i in range(0, MAXTHREADS):
-        # TEST DATA
-        # filename = "test" + str(i) + CHUNk
-        # DEPLOYMENT DATA
-        filename = str(i) + CHUNK
-        _logger.info(f"Creating worker thread no: {i}")
-        threads.append(
-            Thread(target=producer, args=(work_q, control_q, filename, _logger))
-        )
+    t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
+    logger.info("Consumer thread created")
+    threads.append(t_cons)
+    for i in range(0,MAXTHREADS):
+        #filename = "test" + str(i) + CHUNk
+        filename = "40" + CHUNk
+        #filename = str(i) + CHUNk
+        logger.info(f"Creating worker thread no: {i}")
+        threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
     for worker in threads:
         worker.start()
     for worker in threads:
@@ -172,29 +84,23 @@ def search_for_doi(doi, live_results, _logger):
     return result_list
 
 
+
 if __name__ == "__main__":
     import logging
-
     logger = logging.getLogger()
     logger.setLevel("DEBUG")
     h1 = logging.StreamHandler()
     logger.addHandler(h1)
     logger.info("TEST RUN")
     live_result = []
-    logger.info(
-        search_for_doi(
-            [
-                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
-                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
-                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
-                ("10.2165/00128415-200309690-00017", "DATA"),
-                ("10.5772/48313", "DATA"),
-                ("10.15803/ijnc.7.2_419", "DATA"),
-                ("10.1051/0004-6361/201321596e", "DATA"),
-                ("10.2307/40835941", "DATA"),
-            ],
-            live_result,
-            logger,
-        )
-    )
+    logger.info(search_for_doi([
+        ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
+        ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
+        ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
+        ("10.2165/00128415-200309690-00017","DATA"),
+        ("10.5772/48313","DATA"),
+        ("10.15803/ijnc.7.2_419","DATA"),
+        ("10.1051/0004-6361/201321596e","DATA"),
+        ("10.2307/40835941","DATA"),
+        ], live_result, logger))
     logger.info(live_result)

conjurer_librarian/search_bot2.py

@@ -1,206 +0,0 @@
-"""
-This module contains functions for searching for DOI (Digital Object Identifier)
-in a list of live results.
-It includes a producer-consumer pattern implementation using threads and queues.
-
-Functions:
-- producer: Reads lines from a file and puts them into an output queue.
-- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
-- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
-
-Global Variables:
-- MAXTHREADS: Maximum number of worker threads.
-- DATABASE_PATH: Path to the database files.
-- ENCODING: Encoding of the database files.
-- CHUNK: File name pattern for the database files.
-- _sentinel: Sentinel object used to signal termination.
-- result_list: List to store the search results.
-- WORK_Q_SIZE: Maximum size of the work queue.
-"""
-
-# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
-from multiprocessing import Manager, Process, Queue
-from queue import Empty, Full
-import time
-
-# DATA FOR TEST ONLY
-#MAXTHREADS = 6
-#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
-
-# DEPLOYMENT DATA
-MAXTHREADS = 41
-DATABASE_PATH = r"C:\\Database\\chunks\\"
-
-ENCODING = "utf-8"
-CHUNK = "_chunk.txt"
-_sentinel = None
-WORK_Q_SIZE = 355000
-
-
-def producer(out_q, control_q, filename, _logger):
-    """
-    Produces items from the output queue and puts them into the control queue.
-
-    Args:
-        out_q (Queue): Output queue.
-        control_q (Queue): Control queue.
-        filename (str): Name of the file.
-        _logger: Logger object for logging.
-    """
-    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        print(f"Worker {filename} ")
-        line_no = 0
-        while True:
-            line = operated_file.readline()
-            line_no += 1
-            print(f"\t \t \t W{filename}{line_no}\r", end="")
-
-            if not line:
-                print(f"EOF {filename}")
-                break
-            try:
-                out_q.put([line, line_no, filename], block = True, timeout = 60)
-            except Full:
-                print("queue fulll")
-            try:
-                check = control_q.get(block=False)
-            except Empty:
-                check = False
-            if check is None:
-                print(f"TERM signal received {filename}")
-                control_q.put(check)
-                control_q.close()
-                break
-        print(f"Worker finished: {filename}")
-        out_q.put(_sentinel)
-        out_q.close()
-
-
-
-# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
-    """
-    Consumes items from an input queue and checks if DOI exists in the live results list.
-
-    Args:
-        in_q (Queue): Input queue.
-        control_q (Queue): Control queue.
-        doi (list): List of DOI to search for.
-        live_results (list): List to store the search results.
-        _logger: Logger object for logging.
-    """
-    print(f"Consumer thread started: {no} no")
-    empty_counter = 0
-    alive_no = 0
-    while True:
-        done_check = True
-        try:
-            data = in_q.get(block=True, timeout = 1)
-            empty_counter = 0
-            if data is None:
-                control_dict["sentinels"] += 1
-                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
-            else:
-                empty_counter = 0
-                if alive_no > 9:
-                    alive_no = 0
-                alive_no += 1
-                print(f"C{no}{alive_no}\r", end="")
-                for item in result_list:
-                    if item["DOI"] in data[0] and not item["exists"]:
-                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
-                        item["exists"] = True
-                        live_results.append(item)
-                    done_check = done_check and item["exists"]
-                if done_check:
-                    control_q.put(_sentinel)
-        except Empty:
-            empty_counter += 1
-            time.sleep(1)
-            print(f"Consumer {no} empty")
-            if empty_counter > 5:
-                print("Consumer %s empty lvl 2", no)
-                time.sleep(2)
-            elif empty_counter > 10:
-                print(f"Consumer thread finished {no}")
-                break
-        if control_dict["sentinels"] >= MAXTHREADS:
-            print(f"All workers finished killing {no}")
-            break
-    print("Consumer thread finished %s",no )
-    control_q.close()
-
-
-def search_for_doi(doi, live_results, _logger):
-    """
-    Search for DOI in live_results using _logger for logging.
-
-    Args:
-        doi (list): List of DOI to search for.
-        live_results (list): List to store the search results.
-        _logger: Logger object for logging.
-    """
-    with Manager() as manager:
-        control_dict = manager.dict()
-        control_dict["sentinels"] = 0
-        result_list = []
-        for item in doi:
-            result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
-        result_list_proxy = manager.list(result_list)
-
-        threads = []
-        work_q = Queue(maxsize=WORK_Q_SIZE)
-        control_q = Queue()
-        for i in range(0, MAXTHREADS):
-            # TEST DATA
-            #filename = "test" + str(i) + CHUNK
-            # DEPLOYMENT DATA
-            filename = str(i) + CHUNK
-            print(f"Creating worker thread no: {i}")
-            threads.append(
-                Process(target=producer, args=(work_q, control_q, filename, _logger))
-            )
-        for i in range (0, (len(doi)//1000)+5):
-            t_cons = Process(
-                target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger)
-            )
-            print("Consumer thread created")
-            threads.append(t_cons)
-
-        for worker in threads:
-            worker.start()
-        for worker in threads:
-            worker.join()
-        result_list = list(result_list_proxy)
-        for worker in threads:
-            worker.close()
-        return result_list
-
-
-if __name__ == "__main__":
-    import logging
-
-    logger = logging.getLogger()
-    logger.setLevel("DEBUG")
-    h1 = logging.StreamHandler()
-    logger.addHandler(h1)
-    print("TEST RUN")
-    live_result = []
-    print(
-        search_for_doi(
-            [
-                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
-                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
-                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
-                ("10.2165/00128415-200309690-00017", "DATA"),
-                ("10.5772/48313", "DATA"),
-                ("10.15803/ijnc.7.2_419", "DATA"),
-                ("10.1051/0004-6361/201321596e", "DATA"),
-                ("10.2307/40835941", "DATA"),
-                ("NOT_ON_LIST", "DATA"),
-            ],
-            live_result,
-            logger,
-        )
-    )
-    print(live_result)

conjurer_librarian/test_send.py

@@ -1,45 +0,0 @@
-import asyncio
-import json
-import logging
-from json.decoder import JSONDecodeError
-from logging import handlers
-import requests
-
-MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
-SEND_RESULTS = "/conjurer"
-
-async def send_results():
-    logger = logging.getLogger()
-    logger.setLevel(logging.DEBUG)
-    h1 = handlers.RotatingFileHandler(
-        filename="D:\\logs\\librarian.log",
-        encoding="utf-8",
-        mode="a",
-        maxBytes=6 * 1024 * 1024,
-        backupCount=6,
-    )
-
-    logger.addHandler(h1)
-
-    with open("s_results.json", "r+", encoding="utf-8") as s_file:
-        try:
-            database = json.load(s_file)
-        except JSONDecodeError:
-            pass
-    result = database["4bacbca0-ce3b-45c6-90a1-05c70b46b740"]
-    result = {"4bacbca0-ce3b-45c6-90a1-05c70b46b740":result}
-
-    coroutine = asyncio.to_thread(
-        requests.post,
-        f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
-        json=result,
-        timeout=360,
-    )
-    logger.info("SENT")
-    result = await coroutine
-    logger.info(result.status_code)
-    logger.info("SEND CONFIRMED")
-async def main():
-    await send_results()
-if __name__ == "__main__":
-    asyncio.run(main())

conjurer_musician/conjurer_musician.py

@@ -6,9 +6,7 @@ API endpoints.
 """
 import json
 import logging
-import os
 import re
-import requests
 import threading
 import time
 from datetime import datetime
@@ -16,11 +14,10 @@ from logging import handlers
 from pathlib import Path
 from platform import uname
 from sys import platform
-from flask import Flask, jsonify, redirect, request, send_from_directory, render_template
+
+from flask import Flask, jsonify, redirect, request, send_from_directory
 from waitress import serve
 
-MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
-MUSIC_TRACKER = "/prepped_tracks"
 HOST_ADDRESS = "192.168.1.15"
 PORT_ADDRESS = 5000
 if platform in ("linux", "linux2"):
@@ -31,8 +28,6 @@ if platform in ("linux", "linux2"):
         NETRC_FILE = "/home/mtuszowski/.netrc"
         LOGSTORE = "/home/mtuszowski/conjurer/logs/"
         ENCODING = "utf-8"
-        RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
-        PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
 
     else:
         LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
@@ -40,50 +35,53 @@ if platform in ("linux", "linux2"):
         LOGSTORE = "/home/pi/RetroPie/logs/"
         ENCODING = "utf-8"
         MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
-        PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/"
-        RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
-        PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
 
 
 music_file_list = []
-priority_list = []
+
+
+def create_playlist():
+    """
+    Reads a JSON file containing a playlist and writes the playlist items to a text file.
+
+    The JSON file path is '/home/pi/Conjurer/playlist.json',
+    and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
+
+    Raises:
+        JSONDecodeError: If the JSON file cannot be decoded.
+
+    """
+    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
+        with open(
+            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
+        ) as w_file:
+            try:
+                playlist = json.load(r_file)
+                for item in playlist:
+                    w_file.write(item)
+                    w_file.write("\n")
+            except json.JSONDecodeError:
+                pass
+
 
 def rescan():
     """
     The `rescan` function logs a message, scans for mp3 files in a specified folder,
     and adds them to a list of music files.
     """
-    logger = logging.getLogger("conjurer_musician")
-    logger.info("Rescan triggered")
-
+    logging.info("Rescan triggered")
     for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
         temp_music_file = mp3_item.as_posix()
         if platform == "win32":
             temp_music_file = temp_music_file.replace("/", "\\")
         music_file_list.append(temp_music_file)
-
-    for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"):
-        temp_music_file = mp3_item.as_posix()
-        if platform == "win32":
-            temp_music_file = temp_music_file.replace("/", "\\")
-        priority_list.append(temp_music_file)
-
-    with open(
-            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
-        ) as w_file:
-        try:
-            for item in music_file_list:
-                w_file.write(item)
-                w_file.write("\n")
-        except json.JSONDecodeError:
-            pass
-    with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
-        try:
-            for item in priority_list:
-                w_file.write(item)
-                w_file.write("\n")
-        except json.JSONDecodeError:
-            pass
+    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
+        database = music_file_list
+        # self.app.logger.info("DUMPING DATA")
+        s_file.truncate(0)
+        s_file.seek(0)
+        json.dump(database, s_file)
+    create_playlist()
 
 
 def thread_rescan():
@@ -91,87 +89,13 @@ def thread_rescan():
     The `thread_rescan` function periodically triggers a rescan operation after a specified time
     interval.
     """
-    logger = logging.getLogger("conjurer_musician")
-    logger.info("Starting filesystemupdater")
+    logging.info("Starting filesystemupdater")
     while True:
         time.sleep(60 * 60 * 24)
-        logger.info("Rescan triggered")
+        logging.info("Rescan triggered")
         rescan()
 
 
-def scan_tracks():
-    #Set the filename and open the file
-    logger = logging.getLogger("conjurer_musician")
-
-    file = open(RADIOLOG_PATH,'r')
-    #Find the size of the file and move to the end
-    st_results = os.stat(RADIOLOG_PATH)
-    st_size = st_results[6]
-    file.seek(st_size)
-    st_results1 = os.stat(PERSISTENCE_PATH)
-    prev_st_size1 = st_results[6]
-
-    while 1:
-
-        st_results1 = os.stat(PERSISTENCE_PATH)
-        st_size1 = st_results1[6]
-        if prev_st_size1 != st_size1:
-            while prev_st_size1 != st_size1:
-                prev_st_size1 = st_size1
-                st_results1 = os.stat(PERSISTENCE_PATH)
-                st_size1 = st_results1[6]
-                time.sleep(0.1)
-            file1 = open(PERSISTENCE_PATH, 'r')
-            lines = file1.readlines()
-            result = ["now_playing", lines[2]]
-            file1.close()
-            returned = requests.post(
-                    f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
-                    json=result,
-                    timeout=360)
-            logger.info("SENT")
-            logger.info(returned.status_code)
-            logger.info("SEND CONFIRMED")
-
-        where = file.tell()
-        line = file.readline()
-        if not line:
-            time.sleep(1)
-            file.seek(where)
-        else:
-            if re.match(".*Prepared.*",line):
-                result = None
-                if re.match(".*jingles.*", line):
-                    logger.info("jingles")
-                    logger.info(line) # already has newline
-                    result = ["jingles", line]
-                elif re.match(".*priority.*", line):
-                    logger.info("priority")
-                    logger.info(line) # already has newline
-                    result = ["priority", line]
-                elif re.match(".*hit.*", line):
-                    logger.info("hit")
-                    logger.info(line) # already has newline
-                    result = ["hit", line]
-                elif re.match(".*all_playlist.*", line):
-                    logger.info("all")
-                    logger.info(line) # already has newline
-                    result = ["all", line]
-                elif re.match(".*request.*", line):
-                    logger.info("requests")
-                    logger.info(line) # already has newline
-                    result = ["requests", line]
-                if result:
-                    returned = requests.post(
-                    f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
-                    json=result,
-                    timeout=360)
-                    logger.info("SENT")
-                    logger.info(returned.status_code)
-                    logger.info("SEND CONFIRMED")
-        time.sleep(0.1)
-
-
 app = Flask(__name__)
 
 
@@ -278,9 +202,7 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
                 return_list.append(search_weight[itr])
                 if not return_to_bot:
                     with open(
-                        "/home/pi/Conjurer/priority_queue.playlist",
-                        "r+",
-                        encoding="utf-8",
+                        "/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
                     ) as s_file:
                         s_file.write(search_weight[itr][1])
                 break
@@ -289,7 +211,14 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
         fun_logger.info("Wiele plików do zagrania")
         search_weight.sort(key=lambda x: x[0], reverse=True)
         return_list.extend(search_weight[: int(how_many)])
-    fun_logger.info("Done: %s", return_list)
+        if not return_to_bot:
+
+            with open(
+               "/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
+            ) as s_file:
+                s_file.truncate(0)
+                for item in return_list:
+                    s_file.write(item[1] + "\n")
     return return_list
 
 
@@ -324,37 +253,24 @@ def remove_characters(string, character):
 @app.route("/stream", methods=["GET"])
 def stream_music():
     """
-    The function `stream_music` is a route handler for the "/stream" endpoint.
-    It returns a JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
-
-    :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
     """
+    return send_from_directory("/tmp/hls", "stream.m3u8")
 
-    #return send_from_directory("/tmp/hls", "stream.m3u8")
-    return render_template("/home/pi/Conjurer/stream.html")
-
-@app.route("/<string:file_name>")
+@app.route('/<string:file_name>')
 def stream(file_name):
-    """
-    Stream the specified file from the video directory.
-
-    Args:
-        file_name (str): The name of the file to be streamed.
-
-    Returns:
-        Response: The response containing the streamed file.
-    """
-    # trunk-ignore(bandit/B108)
-    video_dir = "/tmp/hls"
+    video_dir = '/tmp/hls'
     return send_from_directory(video_dir, file_name)
 
+
 @app.route("/stream_mp3", methods=["GET"])
 def stream_music_mp3():
     """
-    The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint.
-    It redirects the user to the URL "http://www.example.com" with a status code of 302.
-
-    :return: A redirect response to "http://www.example.com" with a status code of 302.
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
     """
     return redirect("http://www.example.com", code=302)
 
@@ -362,18 +278,19 @@ def stream_music_mp3():
 @app.route("/clear_pr_pls", methods=["GET"])
 def clear_pr_pls():
     """
-    The function `clear_pr_pls` clears the contents of the priority queue playlist file.
-
-    :return: A JSON response indicating the success of the operation.
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
     """
-    app.logger.info("CLEARING PLAYLIST")
-    with open('/home/pi/Conjurer/priority_queue.playlist', 'w', encoding='utf-8') as cleared_pl:
-        cleared_pl.write('')
-
+    with open(
+        "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
+        ) as s_file:
+        s_file.truncate(0)
     return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
     return return_data, 200
 
 
+
 @app.route("/mp3", methods=["GET"])
 def get_music_list():
     """
@@ -408,22 +325,20 @@ def update_music_list():
 @app.route("/get_music", methods=["POST"])
 def look_for_playlist():
     """
-    The function `look_for_playlist` receives a POST request with a JSON payload, logs the received
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
     item, adds it to a music file list, and returns a success message along with the updated record.
-
-    :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`,
-             `message`, `statusCode`, and `data`, with values indicating the success of the operation and the
-             data that was received and added to the `music_file_list`. The status code returned is 200,
-             indicating a successful response.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
     """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
     app.logger.info(record["UUID"])
     app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(
-        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True
-    )
+    return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True)
 
     return_data = (
         jsonify(isError=False, message="Success", statusCode=200, data=return_data),
@@ -431,89 +346,26 @@ def look_for_playlist():
     )
     return return_data
 
-
-@app.route("/request_radio_file", methods=["POST"])
-def add_request():
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info(record["lista_slow"])
-    app.logger.info(record["UUID"])
-    return_data = wyszukaj(
-        record["lista_slow"], 0, app.logger, False
-    )
-
-    with open(
-        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
-    ) as s_file:
-        for item in return_data:
-            s_file.write(item[1] + "\n")
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
-        200,
-    )
-    return return_data
-
-@app.route("/create_priority_playlist", methods=["POST"])
-def create_priority_playlist():
-    """
-    The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received
-    item, adds it to a music file list, and returns a success message along with the updated record.
-
-    :return: A tuple containing a JSON response and a status code.
-        The JSON response includes keys `isError`,
-        `message`, `statusCode`, and `data`, with values
-        indicating the success of the operation and the
-        data that was received and added to the `music_file_list`.
-        The status code returned is 200,indicating a successful response.
-    """
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info(record["lista_slow"])
-    app.logger.info(record["UUID"])
-    app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(
-        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
-    )
-    with open(
-        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
-    ) as s_file:
-        for item in return_data:
-            s_file.write(item[1] + "\n")
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
-        200,
-    )
-    return return_data
-
-
 @app.route("/add_to_priority", methods=["POST"])
 def add_to_priority():
     """
-    The function `add_to_priority` receives a POST request with a JSON payload, logs the received
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
     item, adds it to a music file list, and returns a success message along with the updated record.
-
-    :return: A tuple containing a JSON response and a status code.
-        The JSON response includes keys `isError`,
-        `message`, `statusCode`, and `data`, with values
-        indicating the success of the operation and the
-        data that was received and added to the `music_file_list`.
-        The status code returned is 200,indicating a successful response.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
     """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
     app.logger.info(record["UUID"])
     app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(
-        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
-    )
-    with open(
-        "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8"
-    ) as s_file:
-        for item in return_data:
-            s_file.write(item[1] + "\n")
+    return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False)
+
     return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
         200,
     )
     return return_data
@@ -530,18 +382,8 @@ def flask_debug():
 
 def waitress_run():
     """
-    The `waitress_run` function serves the `app` on host HOST_ADDRESS
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
     and port 5000 using the Waitress WSGI server.
-
-    This function starts the Waitress server and listens for incoming requests
-    on the specified host and port. It uses the `serve` function from the Waitress
-    library to handle the serving process.
-
-    Parameters:
-        None
-
-    Returns:
-        None
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
@@ -565,10 +407,9 @@ if __name__ == "__main__":
     rescan()
     logger.info("Started")
     threads = []
-    # threads.append(threading.Thread(target=flask_debug))
+    #threads.append(threading.Thread(target=flask_debug))
     threads.append(threading.Thread(target=waitress_run))
     threads.append(threading.Thread(target=thread_rescan))
-    threads.append(threading.Thread(target=scan_tracks))
 
     for worker in threads:
         worker.start()

conjurer_musician/music_utils.py

@@ -1,26 +0,0 @@
-# This Python file uses the following encoding: utf-8
-"""This module contains utility scripts used by musi radiostation Conjurer"""
-import time
-import re
-
-NAME = "/home/pi/Conjurer/persistence.log"
-
-def print_top():
-    """
-    Prints the top 65 lines from a file specified by the NAME variable.
-    """
-    while True:
-        line = ""
-        buffer = ""
-        with open(NAME, "r", encoding="utf-8") as f:
-            for _ in range (45):
-                line = f.readline()
-                if re.match(".*mp3", line):
-                    buffer += line
-            print("=====================================")
-            print(buffer)
-            print("=====================================")
-        time.sleep(180)
-
-if __name__ == "__main__":
-    print_top()

conjurer_musician/pl.config

@@ -1,14 +0,0 @@
-$dodaj_do_ulubionych 100 nightwish
-$dodaj_do_ulubionych 100 amon amarth
-$dodaj_do_ulubionych 100 tactical sekt
-$dodaj_do_ulubionych 100 lacuna coil
-$dodaj_do_ulubionych 100 sefa
-$dodaj_do_ulubionych 100 frenchcore
-$dodaj_do_ulubionych 100 peacock
-$dodaj_do_ulubionych 100 vavamuffin
-$dodaj_do_ulubionych 100 abradab
-$dodaj_do_ulubionych 100 eluveitie
-$dodaj_do_ulubionych 100 youtube
-
-$dodaj_do_ulubionych 100 elyose
-$dodaj_do_ulubionych 1 artisans du chaos

conjurer_musician/radio_conjurer.liq

@@ -1,167 +1,48 @@
-# FILEPATH: /home/mtuszowski/conjurer/conjurer_musician/radio_conjurer.liq
-
-# This script sets up a Liquidsoap radio stream with various features and configurations.
-
-# Load icecast credentials from a JSON file
 let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentials.json")
-
-# Enable replaygain metadata processing
 enable_replaygain_metadata()
 
-# Set up a playlog for tracking played tracks
+s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist"))
+s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist"))
 
-l = playlog(duration = 72000.0, persistency="/home/pi/Conjurer/persistence.log")
+s = random(id="randomizer", weights=[7000, 100], [s2, s1])
 
-# Function to check if a track can be played based on its metadata
-def check(r)
-    m = request.metadata(r)
-    if l.last(m) < 36000. then
-        log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).")
-        false
-    else
-        l.add(m)
-        true
-    end
-end
-
-# Define playlists to be used in the stream
-s1 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/all_playlist.playlist"))
-s2 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/priority_queue.playlist"))
-s3 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/hit.playlist"))
-
-# Create a request queue for user-generated requests
-requests_queue = request.queue()
-
-# Function to process the request queue and add new requests
-def queue_processing()
-    text=file.lines("/home/pi/Conjurer/request.playlist")
-    if text != [] then
-        list.iter(fun(item) -> requests_queue.push.uri(item), text)
-        file.remove("/home/pi/Conjurer/request.playlist")
-        f = file.open("/home/pi/Conjurer/request.playlist", create=true)
-        f.close()
-    end
-end
-
-# Randomly select a playlist with weights
-s4 = random(id="randomizer", weights=[2, 3, 5], [s1, s2, s3])
-
-# Load jingles playlist
-jingles = (playlist(reload_mode="watch", "/home/pi/Conjurer/jingles.playlist"))
-
-# Create the main stream with random playlist and jingles
-s = rotate(id="randomizer", weights=[10, 1], [s4, jingles])
-
-s = crossfade(fade_out=2., fade_in=2., duration=4., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
-
-# Set up an interactive harbor for controlling the stream
 interactive.harbor(port = 9999)
-
-# Set up interactive controls for bass boost
-f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.)
-g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.)
+f = interactive.float("f", description="Frequency", min=0., max=1000.,unit="Hz", 200.)
+g = interactive.float("g", description="Gain", min=0., max=20.,unit="dB", 8.)
 b = bass_boost(frequency=f, gain=g, s)
 s = add([s, b])
-
-# Set up interactive control for main volume
 a = interactive.float("main_volume", min=0., max=20., 1.)
-s = compress.multiband.interactive(bands=7, s)
+s = compress.multiband.interactive(bands=3, s)
 
-# Apply audio processing effects
-s = nrj(normalize(s))
-s = amplify(a, s)
+s = nrj(normalize (s))
+s = crossfade(fade_out=3., fade_in=3., duration=5., s)
+s = amplify(a,s)
 
-mic = buffer(input.pulseaudio()) # Microphone
-mic = gate(threshold=-30., range=-80., mic)
-mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic)
+s = blank.skip(max_blank=2., s)
 
-s = add([s, mic])
-#w = interactive.float("wetness", min=0., max=1., 1.)
-#s = dry_wet(w,s,s2)
+emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
+radio = fallback(id="switcher",track_sensitive = false, [s, emergency]) 
 
+handle_metadata = fun (m) -> begin
+    print(m["title"])
+    print(m["artist"])
+    print(m["filename"])
+end
+radio.on_metadata(handle_metadata)
+interactive.persistent("script.params")
 
-# Skip blank sections in the stream
-s = blank.skip(max_blank=10., s)
-
-# Configure logging settings
 log_to_stdout = true
 log_to_file = true
 logpath = "/home/pi/Conjurer/radio_log.log"
 loglevel = 3
-set("log.stdout", log_to_stdout)
-set("log.level", loglevel)
 
-# Enable logging to file
-set("log.file", log_to_file)
-set("log.file.path", logpath)
+# Enable logging on Standard Output and set logging level.
+set("log.stdout",log_to_stdout)
+set("log.level",loglevel)
 
-# Set up emergency fallback track
-emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
-radio =  fallback(id="switcher2", track_sensitive=false, [s, emergency])
-# Set up an interactive control for skipping tracks
-p = interactive.bool("Skip track", false)
+# Enable logging to file. See variable: logpath
+set("log.file",log_to_file)
+set("log.file.path",logpath)
 
-def http_skip(~protocol, ~data, ~headers, uri)=
-    radio.skip()
-    http.response(code=200,data="Skipped")
-end
-harbor.http.register(port=54321,method="GET","/skip", http_skip)
-
-
-# Function to process the request queue and skip the current track if requested
-def check_skip()
-    if p() then
-        log.info("Skipping current track.")
-        radio.skip()
-        p.set(false)
-    end
-end
-
-# Run the queue processing function every 60 seconds
-thread.run(every=60., queue_processing)
-
-
-# Run the check_skip function every 5 seconds
-thread.run(every=15., check_skip)
-
-
-# Enable persistent script parameters
-interactive.persistent("script.params")
-
-# Configure output formats and destinations
-
-output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
-output.pulseaudio(radio)
-#output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
-# Uncomment the following lines to enable additional output formats
-# output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio)
-# output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio)
-# output.icecast(%aac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="aac-stream", radio)
-# output.icecast(%flac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="flac-stream", radio)
-# output.icecast(%vorbis, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="vorbis-stream", radio)
-# output.icecast(%speex, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="speex-stream", radio)
-# output.icecast(%wav, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="wav-stream", radio)
-# output.icecast(%pcm, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="pcm-stream", radio)
-# output.icecast(%raw, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="raw-stream", radio)
-# output.icecast(%s16l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16l-stream", radio)
-# output.icecast(%s16b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16b-stream", radio)
-# output.icecast(%s24l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24l-stream", radio)
-# output.icecast(%s24b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24b-stream", radio)
-# output.icecast(%s32l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32l-stream", radio)
-# output.icecast(%s32b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32b-stream", radio)
-# output.icecast(%s64l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64l-stream", radio)
-# output.icecast(%s64b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64b-stream", radio)
-# output.icecast(%s128l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128l-stream", radio)
-# output.icecast(%s128b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128b-stream", radio)
-# output.icecast(%s256l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256l-stream", radio)
-# output.icecast(%s256b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256b-stream", radio)
-# output.icecast(%s512l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512l-stream", radio)
-# output.icecast(%s512b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512b-stream", radio)
-# output.icecast(%s1024l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024l-stream", radio)
-# output.icecast(%s1024b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024b-stream", radio)
-# output.icecast(%s2048l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048l-stream", radio)
-# output.icecast(%s2048b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048b-stream", radio)
-# output.icecast(%s4096l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096l-stream", radio)
-# output.icecast(%s4096b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096b-stream", radio)
-# output.icecast(%s8192l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192l-stream", radio)
-# output.icecast(%s8192b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192b-stream", radio)
+output.file.hls("/tmp/hls",[("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))],radio)
+output.icecast(%mp3, host="retropie", port=8000,password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)

conjurer_musician/requirements_musician.txt

@@ -1,3 +1,2 @@
 flask
-waitress
-sshtunnel
+waitress

conjurer_musician/stream.html

@@ -1,9 +0,0 @@
-    <html>
-        <head>
-            <title>HTTP Live Streaming Example</title>
-        </head>
-        <body>
-            <video src="/tmp/hls/stream.m3u8" height="300" width="400">
-            </video>
-        </body>
-    </html>

install_music_bot.sh

@@ -8,17 +8,9 @@ cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjur
 cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
 cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
 touch /home/pi/Conjurer/discord_mus_service.log
-
+touch /home/pi/Conjurer/all_playlist.txt
 touch /home/pi/Conjurer/playlist.json
-touch /home/pi/Conjurer/prio_playlist.json
-
-touch /home/pi/Conjurer/all_playlist.playlist
 touch /home/pi/Conjurer/priority_queue.playlist
-touch /home/pi/Conjurer/hit.playlist
-touch /home/pi/Conjurer/request.playlist
-touch /home/pi/Conjurer/persistence.log
-echo "[]" > /home/pi/Conjurer/persistence.log
-
 touch /home/pi/Conjurer/radio_log.log
 ./env/bin/python3 -m pip install --upgrade pip
 ./env/bin/python3 -m pip install -r requirements_musician.txt

requirements_bot.txt

@@ -12,5 +12,4 @@ spotipy
 tiktoken
 PyNaCl
 flask
-waitress
-clickupython
+waitress

restart_radio.sh

@@ -1,3 +0,0 @@
-sshpass -p raspberry ssh pi@192.168.1.15 << 'ENDSSH'
-sudo reboot
-ENDSSH

s_results.json

@@ -0,0 +1 @@
+{"12345678-1234-5678-1234-567812345678": [{"DOI": "10.1057/9781137435026", "title": ["Queer BDSM Intimacies"], "type": "book", "container": "NO"}]}

settings.json

@@ -204,13 +204,6 @@
             false,
             false
         ],
-        "same plusy\\b": [
-            "Jak na cmentarzu Szefie. Jak na cmentarzu",
-            15.0,
-            0,
-            false,
-            false
-        ],
         "kld\\b": [
             "KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.",
             15.0,

spotify_dl/spotify.py

@@ -239,8 +239,6 @@ def get_item_name(sp, item_type, item_id):
         name = sp.album(album_id=item_id).get("name")
     elif item_type == "track":
         name = sp.track(track_id=item_id).get("name")
-    elif item_type == "artist_top_tracks":
-        name = sp.artist_top_tracks(artist_id=item_id).get("name")
     return sanitize(name)
 
 

utils/update_memory.py

@@ -1,33 +0,0 @@
-import json
-MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
-ENCODING = "utf-8"
-
-messages = [
-    {"role": "user", "content" : "Hammer: All systems operational"},
-    {"role": "assistant", "content": "Aye, Szefie! Wszystkie systemy działają sprawnie. Gotów do dalszej służby! Co podać?"},
-    {"role": "user", "content" : "Hammer: Podaj coś na orzeźwienie i przebudzenie. COś co rozgrzeje moje stare kości po ciężkiej pracy!"},
-    {"role": "assistant", "content": "Z przyjemnością Szefie! Dla Ciebie przygotuję Midzynarodówkę Interstelarną Wolf's Dragoon - wysoce energetyczny koktajl rozgrzewający i pobudzający jednocześnie."+
-                                    "Do przygotowania potrzebuję:"+
-                                    "\n"+
-                                    "60 ml mocnego rumu z Wolfa (lub innego, jeśli wolisz)"+
-                                    "30 ml soku z cytryny"+
-                                    "35 ml domowego syropu z mięty"+
-                                    "15 ml świeżo zmielonej kawy"+
-                                    "\n"+
-                                    "Mieszam wszystko w shakerze z lodem, a potem przelewam przez sitko do kieliszka koktajlowego. W górę serwuję z bigoskiem z miętą i zestem z cytryny. Rozgrzewa, orzeźwia i daje kopa na resztę nocy."+
-                                    "\n"+
-                                    "Co powiesz Szefie, podać Ci taki?"},
-    {"role": "user", "content" : "Hammer: Brzmi jak coś co zrobi robotę! Skal!"},
-]
-
-
-for temp in messages:
-
-    with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
-        # First we load existing data into a dict.
-        file_data = json.load(file_memory)
-        # Join new_data with file_data inside emp_details
-        file_data.append(temp)
-        file_memory.seek(0)
-        # convert back to json.
-        json.dump(file_data, file_memory, indent=4)