Search headless (#8)

* Fix

* Final fix to headless

* test

* Fix

* Ready for deployment

bot.py

@@ -693,17 +693,19 @@ async def check_data_q():
                 ctx = fresh_data.ctx
                 message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}'
             else:
-                ctx = client.get_channel(1062047367337095268)
+                ctx = client.get_channel(1062047571557744721)
                 message += f' Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał'
 
 
             message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
             if len(entries)<1:
                 message += "niestety nie ma nic"
+                await ctx.send(message)
             elif len(entries)>=1 and len(entries)<5:
                 message += " znajduje się coś:\n"
                 for item in entries:
                     message+=item
+                await ctx.send(message)
             else:
                 message+= " znajduje się cholernie dużo:\n"
                 for item in entries:
@@ -712,7 +714,6 @@ async def check_data_q():
                         await ctx.send(message)
                         message = ""
 
-            await ctx.send(message)
             #Kept for sentimental reasons
             #await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
     except Empty:
@@ -1064,11 +1065,15 @@ async def get_file(ctx, source, link):
     if source == "Spotify":
         dir_path = None
         item_type, item_id = spotify.parse_spotify_url(link)
-        if item_id:
-            coro = asyncio.to_thread(spotify.fetch_tracks, spotify_ctrl, item_type, link)
-            file_list = await coro
-            coro = asyncio.to_thread(spotify.get_item_name, spotify_ctrl, item_type, item_id)
-            directory_name = await coro
+        if item_type in ["album", "track", "playlist"] and item_id:
+            logger.info(item_type)
+            logger.info(item_id)
+            logger.info("Spotify link provided")
+            file_list = await asyncio.to_thread(spotify.fetch_tracks, spotify_ctrl, item_type, link)
+            logger.info(file_list)
+            directory_name = await asyncio.to_thread(spotify.get_item_name, spotify_ctrl, item_type, item_id)
+            logger.info(directory_name)
+            logger.info("Spotify scrape done")
             url_data = {"urls": []}
             url_dict = {}
             url_dict["save_path"] = Path(
@@ -1078,6 +1083,7 @@ async def get_file(ctx, source, link):
             url_dict["songs"] = file_list
             url_data["urls"].append(url_dict.copy())
             file_name_f = youtube_download.default_filename
+            logger.info("YT-DL")
             coro = asyncio.to_thread(
                 youtube_download.download_songs,
                 songs=url_data,
@@ -1093,6 +1099,7 @@ async def get_file(ctx, source, link):
                 proxy="",
             )
             _ = await coro
+            logger.info("YT-DL done")
             dir_path = (url_dict["save_path"].resolve()).as_posix()
             if platform == "win32":
                 dir_path = dir_path.replace("/", "\\")
@@ -1829,9 +1836,8 @@ async def wyczysc_ulubione(ctx):
     """
     async with ctx.typing():
         coroutine = asyncio.to_thread(
-            requests.post,
+            requests.get,
             f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
-            json={},
             timeout=360,
         )
         _ = await coroutine

communication_subroutine.py

@@ -102,18 +102,19 @@ def scan_incoming(_logger):
         try:
             answer = incoming_q.get(block=False)
             _logger.info("DATA FOUND")
+            record_stored = False
             for record in awaiting_q:
                 if record.uuid in answer.keys():
+                    record_stored = True
+                    record.stop = True
+                    record.entries = answer[record.uuid]
+                    IN_COMM_Q.put(record)
+            if not record_stored:
+                for key in answer.keys():
+                    record = QueryControl("Orphaned", key, "Orphan", _logger, None)
                     record.stop = True
                     record.entries = answer[record.uuid]
                     IN_COMM_Q.put(record)
-                else:
-                    for key in answer.keys():
-                        record = QueryControl("Orphaned", key, "Orphan", _logger, None)
-                        record.stop = True
-                        record.entries = answer[record.uuid]
-                        record.uuid = "000000"
-                        IN_COMM_Q.put(record)
         except Empty:
             time.sleep(1)
 

conjurer_librarian/requirements_librarian.txt

@@ -1,3 +1,3 @@
 habanero
 waitress
-flask
+flask[async]

conjurer_librarian/search_bot.py

@@ -35,7 +35,7 @@ DATABASE_PATH = r"C:\\Database\\chunks\\"
 ENCODING = "utf-8"
 CHUNK = "_chunk.txt"
 _sentinel = object()
-WORK_Q_SIZE = 15500000
+WORK_Q_SIZE = 35500000
 
 
 def producer(out_q, control_q, filename, _logger):
@@ -68,7 +68,7 @@ def producer(out_q, control_q, filename, _logger):
 
 
 # IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _logger):
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
     """
     Consumes items from an input queue and checks if DOI exists in the live results list.
 
@@ -79,29 +79,34 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _log
         live_results (list): List to store the search results.
         _logger: Logger object for logging.
     """
+    _logger.info("Consumer thread started:%s", no)
     for item in doi:
         result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
     while True:
         done_check = True
-        data = in_q.get()
-        if data is _sentinel:
-            _logger.info("Worker finished")
-            control_dict["sentinels"] += 1
-        else:
-            for item in result_list:
-                if item["DOI"] in data and not item["exists"]:
-                    _logger.info(data)
-                    _logger.info("HIT")
-                    item["exists"] = True
-                    live_results.append(item)
-                done_check = done_check and item["exists"]
-            if done_check:
-                control_q.put(_sentinel)
+        try:
+            data = in_q.get(block=False)
+            if data is _sentinel:
+                _logger.info("Worker finished")
+                control_dict["sentinels"] += 1
+            else:
+                for item in result_list:
+                    if item["DOI"] in data and not item["exists"]:
+                        _logger.info(data)
+                        _logger.info("HIT")
+                        item["exists"] = True
+                        live_results.append(item)
+                    done_check = done_check and item["exists"]
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            pass
         if control_dict["sentinels"] >= MAXTHREADS:
             _logger.info("All workers finished")
             break
 
 
+
 def search_for_doi(doi, live_results, _logger):
     """
     Search for DOI in live_results using _logger for logging.
@@ -116,9 +121,9 @@ def search_for_doi(doi, live_results, _logger):
     threads = []
     work_q = Queue(maxsize=WORK_Q_SIZE)
     control_q = Queue()
-    for _ in range (1, len(doi)//1000+1):
+    for i in range (0, (len(doi)//1000)+2):
         t_cons = Thread(
-            target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, _logger)
+            target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)
         )
         _logger.info("Consumer thread created")
         threads.append(t_cons)

conjurer_librarian/test_send.py

@@ -0,0 +1,45 @@
+import asyncio
+import json
+import logging
+from json.decoder import JSONDecodeError
+from logging import handlers
+import requests
+
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+SEND_RESULTS = "/conjurer"
+
+async def send_results():
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    h1 = handlers.RotatingFileHandler(
+        filename="D:\\logs\\librarian.log",
+        encoding="utf-8",
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
+    )
+
+    logger.addHandler(h1)
+
+    with open("s_results.json", "r+", encoding="utf-8") as s_file:
+        try:
+            database = json.load(s_file)
+        except JSONDecodeError:
+            pass
+    result = database["4bacbca0-ce3b-45c6-90a1-05c70b46b740"]
+    result = {"4bacbca0-ce3b-45c6-90a1-05c70b46b740":result}
+
+    coroutine = asyncio.to_thread(
+        requests.post,
+        f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+        json=result,
+        timeout=360,
+    )
+    logger.info("SENT")
+    result = await coroutine
+    logger.info(result.status_code)
+    logger.info("SEND CONFIRMED")
+async def main():
+    await send_results()
+if __name__ == "__main__":
+    asyncio.run(main())

conjurer_musician/conjurer_musician.py

@@ -41,41 +41,6 @@ if platform in ("linux", "linux2"):
 music_file_list = []
 priority_list = []
 
-
-def create_playlist():
-    """
-    Reads a JSON file containing a playlist and writes the playlist items to a text file.
-
-    The JSON file path is '/home/pi/Conjurer/playlist.json',
-    and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
-
-    Raises:
-        JSONDecodeError: If the JSON file cannot be decoded.
-
-    """
-    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
-        with open(
-            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
-        ) as w_file:
-            try:
-                playlist = json.load(r_file)
-                for item in playlist:
-                    w_file.write(item)
-                    w_file.write("\n")
-            except json.JSONDecodeError:
-                pass
-
-    with open("/home/pi/Conjurer/prio_playlist.json", "r+", encoding="utf-8") as r_file:
-        with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
-            try:
-                playlist = json.load(r_file)
-                for item in playlist:
-                    w_file.write(item)
-                    w_file.write("\n")
-            except json.JSONDecodeError:
-                pass
-
-
 def rescan():
     """
     The `rescan` function logs a message, scans for mp3 files in a specified folder,
@@ -95,18 +60,22 @@ def rescan():
             temp_music_file = temp_music_file.replace("/", "\\")
         priority_list.append(temp_music_file)
 
-    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
-        database = music_file_list
-        # self.app.logger.info("DUMPING DATA")
-        s_file.truncate(0)
-        s_file.seek(0)
-        json.dump(database, s_file)
-    with open("/home/pi/Conjurer/prio_playlist.json", "r+", encoding="utf-8") as s_file:
-        # self.app.logger.info("DUMPING DATA")
-        s_file.truncate(0)
-        s_file.seek(0)
-        json.dump(priority_list, s_file)
-    create_playlist()
+    with open(
+            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
+        ) as w_file:
+        try:
+            for item in music_file_list:
+                w_file.write(item)
+                w_file.write("\n")
+        except json.JSONDecodeError:
+            pass
+    with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
+        try:
+            for item in priority_list:
+                w_file.write(item)
+                w_file.write("\n")
+        except json.JSONDecodeError:
+            pass
 
 
 def thread_rescan():
@@ -313,10 +282,10 @@ def clear_pr_pls():
 
     :return: A JSON response indicating the success of the operation.
     """
-    with open(
-        "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
-    ) as s_file:
-        s_file.truncate(0)
+    app.logger.info("CLEARING PLAYLIST")
+    with open('/home/pi/Conjurer/priority_queue.playlist', 'w', encoding='utf-8') as cleared_pl:
+        cleared_pl.write('')
+
     return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
     return return_data, 200
 

conjurer_musician/radio_conjurer.liq

@@ -39,7 +39,6 @@ let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentia
             log.info("no requests")
         end
     end
-    thread.run(every=60., queue_processing)
 
     # Randomly select a playlist with weights
     s = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3])
@@ -68,7 +67,6 @@ let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentia
 
     # Apply audio processing effects
     s = nrj(normalize(s))
-    s = crossfade(fade_out=3., fade_in=3., duration=5., s)
     s = amplify(a, s)
 
     # Skip blank sections in the stream
@@ -90,7 +88,10 @@ let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentia
     emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
 
     # Create a fallback stream with the main stream and emergency track
-    radio = fallback(id="switcher", track_sensitive=true, [requests_queue, s, emergency])
+    cross_s = crossfade(fade_out=3., fade_in=3., duration=5., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
+    radio =  fallback(id="switcher2", track_sensitive=false, [cross_s, emergency])
+
+    thread.run(every=60., queue_processing)
 
     # Handle metadata events
     handle_metadata = fun (m) -> begin

settings.json

@@ -204,6 +204,13 @@
             false,
             false
         ],
+        "same plusy\\b": [
+            "Jak na cmentarzu Szefie. Jak na cmentarzu",
+            15.0,
+            0,
+            false,
+            false
+        ],
         "kld\\b": [
             "KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.",
             15.0,

spotify_dl/spotify.py

@@ -239,6 +239,8 @@ def get_item_name(sp, item_type, item_id):
         name = sp.album(album_id=item_id).get("name")
     elif item_type == "track":
         name = sp.track(track_id=item_id).get("name")
+    elif item_type == "artist_top_tracks":
+        name = sp.artist_top_tracks(artist_id=item_id).get("name")
     return sanitize(name)