Bugfix in script deploy_music.sh

Adding playlists, various bufixes and installation scripts

Fixing

Rename

KUUUURWA

Install script music bot

fx

Fix ?

fix

Synchro

test

test

Undo

tst

test

tst

Undo changes

test

fx

tst

fx

Test commit of two files priority queue.

bot.py

@@ -46,12 +46,7 @@ from spotify_dl import youtube as youtube_download
 
 from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q
 
-# *=========================== Classes definition
 
-# The above code is defining a TypedDict called `Music_Config` in Python. This TypedDict has three
-# keys: "ctx" which is an optional string, "queue" which is a list of strings, and "requester" which
-# is a list of strings. This TypedDict is used to define the structure of a dictionary that should
-# have these specific keys and value types.
 Music_Config = TypedDict(
     "Music_Config",
     {
@@ -63,10 +58,6 @@ Music_Config = TypedDict(
 
 
 class QueryControl:
-    """
-    This class `QueryControl` is used to manage queries with information about the author, UUID,
-    content, logger, context, and replies.
-    """
     def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
         self.author = query_author
         self.uuid = query_uuid
@@ -79,95 +70,6 @@ class QueryControl:
         )
         self.replies = []
 
-class MusicFileList(object):
-    """
-    The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a
-    file service or local directory and update the list with new items.
-    """
-
-    def __init__(self, uplogger) -> None:
-        self.music_file_list = []
-        self.file_service_active = False
-        self.logger = uplogger
-        self.logger.info("Created Playlist organizer class")
-
-    def refresh_file_list(self):
-        """
-        The `refresh_file_list` function attempts to connect to a file service to retrieve a list of music
-        files, and if unsuccessful, it populates the list by scanning a local directory for .mp3 files.
-        """
-        try:
-            self.logger.info("Attempt to connect to file service")
-            response = requests.get(f"{FILE_SERVICE_ADDRESS}{GET_MP3}", timeout=360)
-            self.music_file_list = response.json()["music_file_list"]
-            self.file_service_active = True
-        except requests.exceptions.RequestException as e:
-            for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
-                temp_music_file = mp3_item.as_posix()
-                if platform == "win32":
-                    temp_music_file = temp_music_file.replace("/", "\\")
-                self.music_file_list.append(temp_music_file)
-                self.file_service_active = False
-            self.logger.error(e.strerror)
-            self.logger.error("Service Unavailable")
-
-    def get_file_list(self):
-        """
-        The `get_file_list` function returns the list of music files stored in the object.
-        :return: The `music_file_list` attribute is being returned.
-        """
-        return self.music_file_list
-
-    def update_file_list(self, item):
-        """
-        The `update_file_list` function appends an item to a music file list and sends a POST request with
-        the item data to a file service address.
-
-        :param item: The `item` parameter in the `update_file_list` method is the file that you want to add
-        to the `music_file_list` attribute of the class instance. It is then sent as a JSON payload in a
-        POST request to a file service address along with the endpoint `SEND_MP3`
-        """
-        self.music_file_list.append(item)
-        post_data = {"item": str(item)}
-        requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
-
-class DoSearchView(discord.ui.View):
-    def __init__(self, _uuid, result_list, *args, **kwargs) -> None:
-        super().__init__(*args, **kwargs)
-        self.uuid = _uuid
-        self.result_list = result_list
-
-    @discord.ui.button(
-        label="Nastepna strona", style=discord.ButtonStyle.primary, row=0
-    )
-    async def next_callback(self, interaction, button):
-        # emoji = await ctx.guild.fetch_emoji(id)
-        # button.emoji=1102174413887111189
-        logger.info("Pressed")
-        await interaction.response.edit_message(
-            content="You clicked the button!"
-        )  # Send a message when the button is clicked
-
-    @discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0)
-    async def reset_callback(self, interaction, button):
-        # emoji = await ctx.guild.fetch_emoji(id)
-        # button.emoji=1102174413887111189
-        logger.info("Pressed")
-        await interaction.response.edit_message(
-            content="You clicked the button!"
-        )  # Send a message when the button is clicked
-
-    @discord.ui.button(
-        label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0
-    )
-    async def perv_callback(self, interaction, button):
-        # emoji = await ctx.guild.fetch_emoji(id)
-        # button.emoji=1102174413887111189
-        logger.info("Pressed")
-        await interaction.response.edit_message(
-            content="You clicked the button!"
-        )  # Send a message when the button is clicked
-
 
 # *=========================================== Predefines
 formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -191,8 +93,6 @@ FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
 GET_MP3 = "/mp3"
 SEND_MP3 = "/update_mp3"
 GET_PLAYLIST = "/get_music"
-ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
-CLEAR_PRIO = "/clear_pr_pls"
 LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
 SEND_QUERY = "/query"
 TIME_BETWEEN_CALLS = 100000
@@ -293,6 +193,60 @@ logger.info("Playlist generation")
 
 # *=========================================== Load Data files
 
+
+class MusicFileList(object):
+    """
+    The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a
+    file service or local directory and update the list with new items.
+    """
+
+    def __init__(self, uplogger) -> None:
+        self.music_file_list = []
+        self.file_service_active = False
+        self.logger = uplogger
+        self.logger.info("Created Playlist organizer class")
+
+    def refresh_file_list(self):
+        """
+        The `refresh_file_list` function attempts to connect to a file service to retrieve a list of music
+        files, and if unsuccessful, it populates the list by scanning a local directory for .mp3 files.
+        """
+        try:
+            self.logger.info("Attempt to connect to file service")
+            response = requests.get(f"{FILE_SERVICE_ADDRESS}{GET_MP3}", timeout=360)
+            self.music_file_list = response.json()["music_file_list"]
+            self.file_service_active = True
+        except requests.exceptions.RequestException as e:
+            for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+                temp_music_file = mp3_item.as_posix()
+                if platform == "win32":
+                    temp_music_file = temp_music_file.replace("/", "\\")
+                self.music_file_list.append(temp_music_file)
+                self.file_service_active = False
+            self.logger.error(e.strerror)
+            self.logger.error("Service Unavailable")
+
+    def get_file_list(self):
+        """
+        The `get_file_list` function returns the list of music files stored in the object.
+        :return: The `music_file_list` attribute is being returned.
+        """
+        return self.music_file_list
+
+    def update_file_list(self, item):
+        """
+        The `update_file_list` function appends an item to a music file list and sends a POST request with
+        the item data to a file service address.
+
+        :param item: The `item` parameter in the `update_file_list` method is the file that you want to add
+        to the `music_file_list` attribute of the class instance. It is then sent as a JSON payload in a
+        POST request to a file service address along with the endpoint `SEND_MP3`
+        """
+        self.music_file_list.append(item)
+        post_data = {"item": str(item)}
+        requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
+
+
 music_file_list = []
 music_file_list = MusicFileList(logger)
 music_file_list.refresh_file_list()
@@ -329,8 +283,6 @@ for key in word_reactions:
     word_reactions[key][2] = datetime.now()
 logger.info("Done")
 
-
-
 # *=========================================== Create Client
 logger.info("Creating discord bot")
 client = commands.Bot(intents=intents, command_prefix="$")
@@ -390,8 +342,8 @@ async def on_message(message):
     if isinstance(message.channel, discord.DMChannel):
         channel = client.get_channel(1064888712565100614)
         await channel.send("Słyszałem ja żem że: " + message.content)
-        return
+    else:
-    channel = message.channel
+        channel = message.channel
     await client.process_commands(message)
 
     message.content = message.content.lower()
@@ -730,40 +682,10 @@ async def check_music():
 @tasks.loop(seconds=3)
 # trunk-ignore(pylint/R0915)
 async def check_data_q():
+    channel = client.get_channel(1062047367337095268)
     try:
         data = IN_COMM_Q.get(block=False)
-        entries = []
+        await channel.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
-        if data.stop:
-            searcher = data.author
-            ctx = data.ctx
-            query = data.content
-            l_p = 1
-            for doi in data.entries:
-                logger.info(doi)
-                desc = data.entries[doi]
-                title = desc['Title'][0]
-                entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n")
-                l_p+=1
-            message = '*Z podłogi wysuwa się winda na książki*'
-            message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query}'
-            message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
-            if len(entries)<1:
-                message += "niestety nie ma nic"
-            elif len(entries)>=1 and len(entries)<5:
-                message += " znajduje się coś:\n"
-                for item in entries:
-                    message+=item
-            else:
-                message+= " znajduje się cholernie dużo:\n"
-                for item in entries:
-                    message+=item
-                    if len(message) > 1500:
-                        await ctx.send(message)
-                        message = ""
-
-            await ctx.send(message)
-            #Kept for sentimental reasons
-            #await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
     except Empty:
         pass
 
@@ -1125,10 +1047,8 @@ async def get_file(ctx, source, link):
         dir_path = None
         item_type, item_id = spotify.parse_spotify_url(link)
         if item_id:
-            coro = asyncio.to_thread(spotify.fetch_tracks, args=(spotify_ctrl, item_type, link))
+            file_list = spotify.fetch_tracks(spotify_ctrl, item_type, link)
-            file_list = await coro
+            directory_name = spotify.get_item_name(spotify_ctrl, item_type, item_id)
-            coro = asyncio.to_thread(spotify.get_item_name, args=(spotify_ctrl, item_type, item_id))
-            directory_name = await coro
             url_data = {"urls": []}
             url_dict = {}
             url_dict["save_path"] = Path(
@@ -1355,7 +1275,6 @@ async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None):
         json=jrequest,
         timeout=360,
     )
-
     return_data = await coroutine
     if not return_data.status_code == 200:
         await ctx.send("Wołaj szefa - coś się wyjebało")
@@ -1781,63 +1700,6 @@ async def zagraj_mi_kawalek(ctx):
                 "Obawiam się że nie mogę nic znaleźć u siebie. Spróbuj komendy '$dej_co_ze_spotifaja' jeśli masz link"
             )
 
-@client.hybrid_command(
-    name="dodaj_do_ulubionych",
-    description="Dodaje do listy ulubionych w radiu",
-    guild=discord.Object(id=664789470779932693),
-)
-async def dodaj_do_ulubionych(ctx):
-    """
-    Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
-    Rest of the line are search terms.
-
-    :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
-    represents the context in which the command was invoked, including information such as the message,
-    the channel, the server, and the user who invoked the command. This parameter is required in all
-    Discord.py commands
-    """
-    async with ctx.typing():
-        logger.info("Zaczynam szukać timestamp %s", datetime.now())
-
-        dlugosc_playlisty = ctx.message.content.split()[1]
-        word_list = ctx.message.content.split()
-        jrequest = {
-            "lista_slow": word_list,
-            "dlugosc_plejlisty": dlugosc_playlisty,
-            "UUID": str(uuid.uuid4()),
-        }
-        coroutine = asyncio.to_thread(
-            requests.post,
-            f"{FILE_SERVICE_ADDRESS}{ADD_TO_PRIO_PLAYLIST}",
-            json=jrequest,
-            timeout=360,
-        )
-        _ = await coroutine
-
-@client.hybrid_command(
-    name="wyczysc_ulubione",
-    description="Czysci liste ulubionych w radiu",
-    guild=discord.Object(id=664789470779932693),
-)
-async def wyczysc_ulubione(ctx):
-    """
-    Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
-    Rest of the line are search terms.
-
-    :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
-    represents the context in which the command was invoked, including information such as the message,
-    the channel, the server, and the user who invoked the command. This parameter is required in all
-    Discord.py commands
-    """
-    async with ctx.typing():
-        coroutine = asyncio.to_thread(
-            requests.post,
-            f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
-            json={},
-            timeout=360,
-        )
-        _ = await coroutine
-
 
 @client.hybrid_command(
     name="zrob_mi_plejliste",
@@ -2203,6 +2065,44 @@ async def parametry_muzyki_mego_ludu(
                 logger.info(exce)
 
 
+class DoSearchView(discord.ui.View):
+    def __init__(self, uuid, result_list, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.uuid = uuid
+        self.result_list = result_list
+
+    @discord.ui.button(
+        label="Nastepna strona", style=discord.ButtonStyle.primary, row=0
+    )
+    async def next_callback(self, interaction, button):
+        # emoji = await ctx.guild.fetch_emoji(id)
+        # button.emoji=1102174413887111189
+        logger.info("Pressed")
+        await interaction.response.edit_message(
+            content="You clicked the button!"
+        )  # Send a message when the button is clicked
+
+    @discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0)
+    async def reset_callback(self, interaction, button):
+        # emoji = await ctx.guild.fetch_emoji(id)
+        # button.emoji=1102174413887111189
+        logger.info("Pressed")
+        await interaction.response.edit_message(
+            content="You clicked the button!"
+        )  # Send a message when the button is clicked
+
+    @discord.ui.button(
+        label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0
+    )
+    async def perv_callback(self, interaction, button):
+        # emoji = await ctx.guild.fetch_emoji(id)
+        # button.emoji=1102174413887111189
+        logger.info("Pressed")
+        await interaction.response.edit_message(
+            content="You clicked the button!"
+        )  # Send a message when the button is clicked
+
+
 @client.hybrid_command(
     name="wyszukaj_linki_do_dokumentow",
     description="Szuka linkow doi w bazie crossref i podaje linki do scihuba",
@@ -2211,8 +2111,8 @@ async def parametry_muzyki_mego_ludu(
 async def wyszukaj_linki_do_dokumentow(ctx):
     query = ctx.message.content
     query_uuid = uuid.uuid4()
-    #TODO: TESTING ONLY!!
+    # query_uuid = "test"
-    #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
+    # 3 wyslij Librariana
     json_query = {"UUID": str(query_uuid), "query": str(query)[30:], "page": 1}
     coroutine = asyncio.to_thread(
         requests.post,
@@ -2245,6 +2145,46 @@ async def wyszukaj_linki_do_dokumentow(ctx):
     await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+
     " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.")
 
+'''
+async def wyswietl_wyniki_wyszukiwania(ctx):
+
+    # 4 zwroc 4 wyniki od librariana
+    # 5 przyciski przy każdym wierszu pozwalają sprawdzić czy plik istnieje na scihubie - jeśli tak podaje link do sciagniecia
+    # 6 ostatni wiersz to dalej, wróć, skasuj, zaciagnij kolejne wyniki (jeśli jest ich więcej niż 1000)
+    result_view = DoSearchView(
+        uuid=str(query_uuid), result_list=query_response.json()["data"]
+    )
+    resulttab = []
+    resultlink = []
+    hit, fetched, total, _ = query_response.json()["data"][0]
+    answer = query_response.json()["data"][1]
+
+    for item in answer:
+        resulttab.append(str(item["title"]))
+        item_link = "https://sci-hub.se/" + str(item["DOI"])
+        resultlink.append(item_link)
+        logger.info(len(resulttab))
+    for i in range(0, min(4, len(resulttab))):
+        logger.info(i)
+        logger.info(resulttab[i])
+        logger.info(resultlink[i])
+        url = resultlink[i]
+        logger.info(url)
+        temp_label = str(resulttab[i])
+        temp_label = temp_label.strip("[']")
+        result_view.add_item(
+            discord.ui.Button(label=str(temp_label[0:79]), row=i + 1, url=url)
+        )
+
+    labelka = (
+        "Wyniki wyszukiwania dla: "
+        + str(query)[30:]
+        + f"\nWszystkich {total} sprawdzonych: {fetched} znalezionych: {hit} "
+    )  # ile znaleziono artykułów w ogóle, ile zostało po refine
+    await ctx.send(
+        labelka, view=result_view
+    )  # Send a message with our View class that contains the button
+'''
 
 @client.hybrid_command(
     name="krecimy_pornola",
@@ -2283,6 +2223,7 @@ async def krecimy_pornola(ctx):
 
 # *================================== Run
 if __name__ == "__main__":
+    # TODO: wpakowac kondzia w wątki odpalic asynchronicznie rownolegle z flaskiem do sterowania nim
     logger.info("Starting discord bot")
     threads = []
     logger.info("Starting discord bot: Creating threads")

communication_subroutine.py

@@ -1,7 +1,6 @@
 import logging
 import threading
 import json
-import time
 from queue import Queue, Empty
 from flask import Flask, jsonify, request
 from waitress import serve
@@ -10,34 +9,28 @@ HOST_ADDRESS = "192.168.1.191"
 PORT_ADDRESS = 5000
 OUT_COMM_Q = Queue()
 IN_COMM_Q = Queue()
-awaiting_q = []
+internal_q = Queue()
-incoming_q = Queue()
 app = Flask(__name__)
 
 @app.route("/conjurer", methods=["POST"])
 def answer_external_command():
-    """
-    The function `answer_external_command` logs the request data, loads the data as JSON, logs the
-    record, and then puts the record into an incoming queue before returning a success message.
-    :return: The function `answer_external_command()` is returning a JSON response with the message
-    "SUCCESS".
-    """
-    app.logger.info(request)
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info("DATA RECEIVED")
-    incoming_q.put(record)
+
+    
+    while True:
+        try:
+            q_data = internal_q.get(block=False)
+        except Empty:
+            break
+    if q_data.uuid in record:
+        pass
+
     return jsonify("SUCCESS")
 
 @app.route("/conjurer", methods=["GET"])
 def check_alive():
-    """
-    The function "check_alive" returns a JSON response with the message "ALIVE" when the "/conjurer"
-    route is accessed via a GET request.
-    :return: The code snippet is a Flask route that listens for GET requests to the "/conjurer"
-    endpoint. When a GET request is made to this endpoint, the function check_alive() is called, which
-    returns a JSON response with the message "ALIVE".
-    """
     return jsonify("ALIVE")
 
 def flask_debug(_logger):
@@ -59,58 +52,19 @@ def waitress_run(_logger):
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
 def scan_queue(_logger):
-    """
-    The function `scan_queue` reads data from a queue, logs it, and appends it to another queue.
-
-    :param _logger: The `_logger` parameter is typically an instance of a logging object that is used to
-    record and store log messages. It is commonly used to track the flow of the program, record errors,
-    and provide information for debugging purposes. In this code snippet, the `_logger` object is used
-    to log the
-    """
     while True:
         data = OUT_COMM_Q.get()
         _logger.info(data)
-        awaiting_q.append(data)
+        internal_q.put(data)
-
+        query_list.append(data)
-def scan_incoming(_logger):
-    """
-    The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data
-    is found.
-
-    :param _logger: The `_logger` parameter in the `scan_incoming` function is a logger object that is
-    used to log messages or information during the execution of the function. It is typically used for
-    debugging, monitoring, or tracking the flow of the program. In this case, the `_logger` is being
-    used
-    """
-    while True:
-        try:
-            answer = incoming_q.get(block=False)
-            _logger.info("DATA FOUND")
-            for record in awaiting_q:
-                if record.uuid in answer.keys():
-                    record.stop = True
-                    record.entries = answer[record.uuid]
-                    IN_COMM_Q.put(record)
-        except Empty:
-            time.sleep(1)
-
 
 def comm_subroutine(logger):
-    """
-    The `comm_subroutine` function starts multiple threads to run different tasks concurrently.
-
-    :param logger: The `logger` parameter in the `comm_subroutine` function is an instance of a logger
-    object that is used for logging messages at various levels (e.g., debug, info, warning, error). In
-    the provided code snippet, the logger is used to log messages at the "info" level
-    """
     #logger.setLevel(logging.DEBUG)
     logger.info("Started")
     threads = []
-    #threads.append(threading.Thread(target=flask_debug, args=(logger,)))
+    threads.append(threading.Thread(target=flask_debug, args=(logger,)))
-    threads.append(threading.Thread(target=waitress_run, args=(logger,)))
+    #threads.append(threading.Thread(target=waitress_run, args=(logger,)))
     threads.append(threading.Thread(target=scan_queue,args=(logger,)))
-    threads.append(threading.Thread(target=scan_incoming,args=(logger,)))
-
 
     for worker in threads:
         worker.start()

conjurer_librarian/conjurer_librarian.py

@@ -1,33 +1,37 @@
-import re
+import asyncio
+import json
 import logging
+import netrc
+import re
 import threading
 import time
-from flask import Flask, jsonify, request
-import netrc
-import json
-import asyncio
 from json.decoder import JSONDecodeError
+from queue import Queue
+from typing import Callable
 from urllib.request import urlopen
+from uuid import UUID
+
+import requests
+import search_bot
+from flask import Flask, jsonify, request
 from habanero import Crossref
 from waitress import serve
-from queue import Queue
+
-import search_bot
-import scrape_bot
-import requests
 NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
 
 HOST_ADDRESS = "192.168.1.192"
 PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
-BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
+BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
-MAX_CR_RESULTS = 150000
+
 ENCODING = "utf-8"
 app = Flask(__name__)
 
 librarian_queue = Queue()
 librarian_list = []
 
+
 class Librarian(object):
 
     def __init__(self, app, query, uuid) -> None:
@@ -36,37 +40,36 @@ class Librarian(object):
         self.cr = Crossref(mailto=authTokens[0])
         self.query = query
         self.uuid = str(uuid)
-        self.limit = MAX_CR_RESULTS
+        self.limit = 3000
         self.fetched = 0
         self.hit = 0
         self.total = 0
         self.app = app
         self.logger = app.logger
         self.live_results = []
-        self.final_result = []
-        self.not_in_db = []
         self.search_result_from_cr = {}
         self.done = False
 
     async def search_crossref(self, query):
         self.app.logger.info("STARTED SEARCH")
-        result = self.cr.works(query = query, limit = 1000)
+        result = self.cr.works(query=query, limit=1000)
         self.search_result_from_cr.update(result)
-        self.total = result['message']['total-results']
+        self.total = result["message"]["total-results"]
-        self.fetched += len(result['message']['items'])
+        self.fetched += len(result["message"]["items"])
-        self.logger.info (self.total)
+        self.logger.info(self.total)
         self.logger.info(self.fetched)
         while self.total > self.fetched and self.limit > self.fetched:
-            result = self.cr.works(query = query, limit = 1000)
+            result = self.cr.works(query=query, limit=1000)
             self.search_result_from_cr.update(result)
-            self.total = result['message']['total-results']
+            self.total = result["message"]["total-results"]
-            self.fetched += len(result['message']['items'])
+            self.fetched += len(result["message"]["items"])
-            self.logger.info (self.total)
+            self.logger.info(self.total)
             self.logger.info(self.fetched)
-            time.sleep(0.1)
+
+            time.sleep(1)
         self.app.logger.info("CROSSREF DONE")
-        #TODO: add to sql with UUID
+        # TODO: add to sql with UUID
-        #sql row - title, doi, author, uuid, last retrieved, is_available
+        # sql row - title, doi, author, uuid, last retrieved, is_available
 
         with open("rr_results.json", "r+", encoding="utf-8") as data_file:
             # First we load existing data into a dict.
@@ -76,13 +79,29 @@ class Librarian(object):
                 file_data = {}
             summarized_results = []
             self.app.logger.info("REFINE: Removing all derived works from the list")
-            for item in self.search_result_from_cr['message']['items']:
+            for item in self.search_result_from_cr["message"]["items"]:
                 container = "NO"
                 if "container-title" in item:
                     container = "YES"
                 else:
-                    summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
+                    summarized_results.append(
-            result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
+                        {
+                            "DOI": item["DOI"],
+                            "title": item["title"] if "title" in item else None,
+                            "type": item["type"] if "type" in item else None,
+                            "container": container,
+                        }
+                    )
+            result = {
+                self.uuid: {
+                    "total_results": self.search_result_from_cr["message"][
+                        "total-results"
+                    ],
+                    "on_page": 1,
+                    "summary": summarized_results,
+                    "results": self.search_result_from_cr["message"]["items"],
+                }
+            }
             if file_data:
                 file_data.update(result)
             else:
@@ -96,7 +115,9 @@ class Librarian(object):
 
     async def check_dois_in_local_db(self, doi):
         self.app.logger.info("REFINE: Running search in the backend app")
-        coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
+        coro = asyncio.to_thread(
+            search_bot.search_for_doi, doi, self.live_results, self.app.logger
+        )
         result = await coro
         return result
 
@@ -109,25 +130,34 @@ class Librarian(object):
             data = response.read()
             text = data.decode("utf-8")
             for line in text.splitlines():
-                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                if re.match(
+                    r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
+                    line,
+                ):
                     return False
-                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
+                if m := re.match(
+                    r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
+                ):
                     direct_download_link = "https:" + str(m.group(1))
-                    self.app.logger.info (direct_download_link)
+                    self.app.logger.info(direct_download_link)
                     return True
         return False
 
-    async def check_if_exists(self, doi, brute_force = False):
+    async def check_if_exists(self, doi, brute_force=False):
         result = {}
         result = await self.check_dois_in_local_db(doi)
         if brute_force and not result:
             for item in doi:
                 item_link = "https://sci-hub.se/" + item[0]
-                tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
+                tmp = {
+                    "DOI": item["DOI"],
+                    "exists": await self.check_if_exists_brute_force(item_link),
+                    "data": item,
+                }
                 result.update(tmp)
         return result
 
-    async def refine_query(self, unrefined_reqult, brute_force = False):
+    async def refine_query(self, unrefined_reqult, brute_force=False):
         temp = []
         refined_result = []
         for key in unrefined_reqult:
@@ -142,69 +172,56 @@ class Librarian(object):
             temp.append([item["DOI"], item])
 
         refined_result = await self.check_if_exists(temp)
-
+        temp = []
         for item in refined_result:
             if item["exists"]:
-                self.final_result.append(item["data"])
+                temp.append(item["data"])
-            else:
+        refined_result = temp
-                self.not_in_db.append(item["data"])
+        self.hit = len(refined_result)
-        self.hit = len(self.final_result)
+        return refined_result
-        return self.final_result
+
-    #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
+    # TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
     async def answer_query(self):
-        #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
+        # TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
-        #Search sql for UUID
+        # Search sql for UUID
-        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        # SQL SELECT * FROM RESULTS WHERE UUID = uuid
         self.logger.info(f"Search started {self.uuid}")
+        uuid_found = False
         database = None
         query = self.query
         with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
-            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
+            # tu logika komunikacji z SQL i cała logika związana z wyszukaniem
-            #ale tymczasowo plik
+            # ale tymczasowo plik
             try:
-                database =  json.load(cr_file)
+                database = json.load(cr_file)
             except JSONDecodeError:
                 pass
-            #TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
+
             if database:
                 for item in database.keys():
                     if item in self.uuid:
                         self.result = await self.refine_query(database)
-                        temp = {}
-                        for item in self.result:
-                            temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
-                        self.result = temp
-                        temp2 = {}
-                        for item in self.not_in_db:
-                            temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
-                        self.not_in_db = temp2
                         return self.result
                 answer = await self.search_crossref(query=query)
                 database.update(answer)
                 cr_file.truncate(0)
                 cr_file.seek(0)
-                json.dump(database,cr_file)
+                json.dump(database, cr_file)
                 self.result = await self.refine_query(answer)
+                return self.result
             else:
                 answer = await self.search_crossref(query=query)
                 database = answer
                 cr_file.truncate(0)
                 cr_file.seek(0)
-                json.dump(database,cr_file)
+                json.dump(database, cr_file)
                 self.result = await self.refine_query(answer)
                 self.app.logger.info("Refined")
                 self.done = True
-            temp = {}
+                return self.result
-            for item in self.result:
-                temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
-            self.result = temp
-            temp2 = {}
-            for item in self.not_in_db:
-                temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
-                self.not_in_db = temp2
-            return self.result
 
-#============================= FLASK INTERNALS===============================
+
+# ============================= FLASK INTERNALS===============================
 def flask_debug():
     """
     The `flask_debug` function starts a Flask application in debug mode without using the reloader.
@@ -213,6 +230,7 @@ def flask_debug():
     # trunk-ignore(bandit/B201)
     app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
+
 def waitress_run():
     """
     The `waitress_run` function serves the `app` on host "0.0.0.0"
@@ -220,6 +238,7 @@ def waitress_run():
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
+
 class BackgroundTaskSearch(threading.Thread):
     def run(self):
         loop = asyncio.new_event_loop()  # loop = asyncio.get_event_loop()
@@ -229,34 +248,18 @@ class BackgroundTaskSearch(threading.Thread):
     async def _run(self):
         while True:
             database = None
-            ndb_database = None
             librarian = librarian_queue.get()
             self.app.logger.info("STARTED")
             result = await librarian.answer_query()
             result = {librarian.uuid: result}
             self.app.logger.info("Saving to file")
-            with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
-                try:
-                    ndb_database =  json.load(ndb_file)
-                except JSONDecodeError:
-                    pass
-                if ndb_database:
-                    ndb_database.update(librarian.not_in_db)
-                else:
-                    ndb_database = librarian.not_in_db
-                ndb_file.truncate(0)
-                ndb_file.seek(0)
-                json.dump(ndb_database, ndb_file)
-
             with open("s_results.json", "r+", encoding="utf-8") as s_file:
                 try:
-                    database =  json.load(s_file)
+                    database = json.load(s_file)
                 except JSONDecodeError:
                     pass
                 if database:
-                    self.app.logger.info(database)
+                    database.extend(result)
-                    self.app.logger.info(result)
-                    database.update(result)
                 else:
                     database = result
                 self.app.logger.info("DUMPING DATA")
@@ -267,18 +270,18 @@ class BackgroundTaskSearch(threading.Thread):
 
             self.app.logger.info(result)
             coroutine = asyncio.to_thread(
-            requests.post,
+                requests.post,
-            f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+                f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
-            json=result,
+                json=result,
-            timeout=360,
+                timeout=360,
-    )       
+            )
             self.app.logger.info("SENT")
-            result = await coroutine
+            _ = await coroutine
-            self.app.logger.info(result.status_code)
             self.app.logger.info("SEND CONFIRMED")
             time.sleep(1)
 
-#==================================SERVER ROUTES==========================================
+
+# ==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
     record = json.loads(request.data)
@@ -286,6 +289,9 @@ async def query_database():
     app.logger.info(record["query"])
     app.logger.info(record["UUID"])
     uuid = record["UUID"]
+    # TODO: TESTING ONLY======================
+    uuid = UUID("{12345678-1234-5678-1234-567812345678}")
+
     cl = Librarian(app, record["query"], uuid)
     librarian_queue.put(cl)
     librarian_list.append(cl)
@@ -296,6 +302,7 @@ async def query_database():
     )
     return return_data
 
+
 @app.route("/get_partial_result", methods=["POST"])
 async def get_partial():
     record = json.loads(request.data)
@@ -311,22 +318,22 @@ async def get_partial():
     )
     return return_data
 
-#=======================================MAIN===================================================
+
+# =======================================MAIN===================================================
 if __name__ == "__main__":
-    logger = app.logger
+    logger = logging.getLogger("conjurer_musician")
     logger.setLevel(logging.DEBUG)
     threads = []
-    threads.append(threading.Thread(target=waitress_run))
+    # threads.append(threading.Thread(target=waitress_run))
-    #threads.append(threading.Thread(target=flask_debug))
+    threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
     threads.append(bgtask)
-    threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,)))
     i = 0
     for worker in threads:
         try:
             logger.info(f"App number: {i}")
-            i+=1
+            i += 1
             worker.start()
         except RuntimeError as e:
             logger.error("Exploded")

conjurer_librarian/run_server.bat

@@ -1 +0,0 @@
-c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py

conjurer_librarian/scrape_bot.py

@@ -1,93 +0,0 @@
-import json
-from json.decoder import JSONDecodeError
-from urllib.request import urlopen
-from requests import Timeout,ConnectTimeout, ConnectionError
-from queue import Queue
-import re
-import time
-import random
-import logging
-from threading import Thread
-SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
-SCR_FILENAME = "40_chunk.txt"
-SCR_ENCODING = "utf-8"
-
-WORK_Q = Queue()
-random.seed()
-def load_ndb_to_q(logger):
-    logger.info("Loader started")
-    while True:    
-        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
-            try:
-                ndb_database =  json.load(ndb_file)
-                for item in ndb_database.keys():
-                    logger.info(item)
-                    url = f"https://sci-hub.se/{item}"
-                    WORK_Q.put([item, url, False])
-                ndb_file.truncate(0)
-            except JSONDecodeError:
-                time.sleep(60*60*2)
-                pass
-        time.sleep(180)
-
-def check_if_exists_brute_force(logger):
-    while True:
-        logger.info("Scraper tick")
-        item = WORK_Q.get()
-        page_url =item[1] 
-        logger.error("REFINE: Brute force search!")
-        if not page_url.startswith(("http:", "https:")):
-            raise ValueError("URL must start with 'http:' or 'https:'")
-        blocked = True
-        try:
-        # trunk-ignore(bandit/B310)
-            with urlopen(page_url) as response:
-                data = response.read()
-                text = data.decode("utf-8")
-                if item[2]:
-                    logger.info("Already found")
-                    time.sleep(180)
-                    blocked = False
-                else:
-                    for line in text.splitlines():
-                        if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
-                            blocked = False
-                            logger.info("Not found")
-                            item[2] = False
-                        if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
-                            blocked = False
-                            direct_download_link = "https:" + str(m.group(1))
-                            logger.info (direct_download_link)
-                            with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
-                                operated_file.write("\n")
-                                operated_file.write(item[0])
-                                item[2] = True
-        except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
-            pass
-        if blocked:
-            logger.info(item)
-            logger.info(text)
-            logger.error("Got blocked. Fuck.")
-            time.sleep(60*60*72)
-        # trunk-ignore(bandit/B311)
-        rand = random.randint(1,100)
-        logger.info(f"Sleeping for {3*rand} minutes")
-        time.sleep(180*rand)
-
-def scraper(logger = None):
-    if not logger:
-        logger = logging.getLogger()
-        logger.setLevel("DEBUG")
-        h1 = logging.StreamHandler()
-        logger.addHandler(h1)
-        logger.info("No logger set. Test run")
-  
-    loader = Thread(target = load_ndb_to_q, args = (logger,))
-    scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
-    loader.start()
-    scraper.start()
-    loader.join()
-    scraper.join()
-
-if __name__ == "__main__":
-    scraper()

conjurer_librarian/search_bot.py

@@ -1,30 +1,25 @@
-#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
-
 from queue import Queue, Empty
 from threading import Thread
 q = Queue()
 
-#TODO: DATA FOR TEST ONLY
+#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy
-#MAXTHREADS = 5
+#MAXTHREADS = 3
-#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+MAXTHREADS = 1
 
-#TODO: DEPLOYMENT DATA
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
-MAXTHREADS = 41
 DATABASE_PATH = r'C:\\Database\\chunks\\'
 
 ENCODING = "utf-8"
 CHUNk = "_chunk.txt"
 _sentinel = object()
 result_list = []
-WORK_Q_SIZE = 15500000
 
-def producer(out_q, control_q, filename, logger):
+#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+
+def producer(out_q, control_q, filename):
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        logger.info(f"Worker {filename} ")
+        lines = operated_file.readlines()
-        while True:
+        for line in lines:
-            line = operated_file.readline()
-            if not line:
-                break
             out_q.put(line)
             try:
                 check = control_q.get(block=False)
@@ -32,11 +27,11 @@ def producer(out_q, control_q, filename, logger):
                 check = False
                 pass
             if check is _sentinel:
-                logger.info("TERM signal received")
+                print("TERM signal received")
                 control_q.put(check)
                 break
-        logger.info(f"Worker finished: {filename}")
+    print(f"Worker finished: {filename}")
-        out_q.put(_sentinel)
+    out_q.put(_sentinel)
  
 #IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
 #TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
@@ -45,6 +40,10 @@ def consumer(in_q, control_q, doi, live_results, logger):
     for item in doi:
         result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
     while True:
+        #if in_q.empty():
+        #    print("Queue empty!")
+        #if in_q.full():
+        #    print("Queue full!")
         done_check = True
         data = in_q.get()
         if data is _sentinel:
@@ -52,7 +51,7 @@ def consumer(in_q, control_q, doi, live_results, logger):
             sentinels += 1
         else:
             for item in result_list:
-                if item["DOI"] in data and not item["exists"]:
+                if item["DOI"] in data:
                     logger.info(data)
                     logger.info("HIT")
                     item["exists"] = True
@@ -67,24 +66,25 @@ def consumer(in_q, control_q, doi, live_results, logger):
 
 def search_for_doi(doi, live_results, logger):
     threads = []
-    work_q = Queue(maxsize=WORK_Q_SIZE)
+    work_q = Queue(maxsize=200000)
     control_q = Queue()
     t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
     logger.info("Consumer thread created")
     threads.append(t_cons)
     for i in range(0,MAXTHREADS):
-        #TODO: TEST DATA
         #filename = "test" + str(i) + CHUNk
-        #TODO: DEPLOYMENT DATA
+        filename = "40" + CHUNk
-        filename = str(i) + CHUNk
+        #filename = str(i) + CHUNk
         logger.info(f"Creating worker thread no: {i}")
-        threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
+        threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
     for worker in threads:
         worker.start()
     for worker in threads:
         worker.join()
     return result_list
 
+
+
 if __name__ == "__main__":
     import logging
     logger = logging.getLogger()

s_results.json

@@ -0,0 +1 @@
+{"12345678-1234-5678-1234-567812345678": [{"DOI": "10.1057/9781137435026", "title": ["Queer BDSM Intimacies"], "type": "book", "container": "NO"}]}