Fixing the bug in the conjurer_musician.py file.

Fx

Fix

fx

Fix ?

Revert "Fix"

This reverts commit 4e185f7b34dfede04ed36ab9813bd9d1f7fb8b35.

Adding deploy for music bot
This commit is contained in:
2024-04-17 12:32:54 +01:00
committed by migatu
parent 0375b0a892
commit ca6896ebcf
7 changed files with 229 additions and 100 deletions
+95 -58
View File
@@ -1,27 +1,29 @@
import re
import asyncio
import json
import logging
import netrc
import re
import threading
import time
from flask import Flask, jsonify, request
import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from queue import Queue
from typing import Callable
from urllib.request import urlopen
from uuid import UUID
import requests
import search_bot
from flask import Flask, jsonify, request
from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
import requests
from typing import Callable
from uuid import UUID
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
ENCODING = "utf-8"
app = Flask(__name__)
@@ -29,6 +31,7 @@ app = Flask(__name__)
librarian_queue = Queue()
librarian_list = []
class Librarian(object):
def __init__(self, app, query, uuid) -> None:
@@ -49,25 +52,25 @@ class Librarian(object):
async def search_crossref(self, query):
self.app.logger.info("STARTED SEARCH")
result = self.cr.works(query = query, limit = 1000)
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.logger.info (self.total)
self.total = result["message"]["total-results"]
self.fetched += len(result["message"]["items"])
self.logger.info(self.total)
self.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched:
result = self.cr.works(query = query, limit = 1000)
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.logger.info (self.total)
self.total = result["message"]["total-results"]
self.fetched += len(result["message"]["items"])
self.logger.info(self.total)
self.logger.info(self.fetched)
time.sleep(1)
self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
# TODO: add to sql with UUID
# sql row - title, doi, author, uuid, last retrieved, is_available
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
# First we load existing data into a dict.
try:
@@ -76,18 +79,34 @@ class Librarian(object):
file_data = {}
summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list")
for item in self.search_result_from_cr['message']['items']:
for item in self.search_result_from_cr["message"]["items"]:
container = "NO"
if "container-title" in item:
container = "YES"
else:
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
summarized_results.append(
{
"DOI": item["DOI"],
"title": item["title"] if "title" in item else None,
"type": item["type"] if "type" in item else None,
"container": container,
}
)
result = {
self.uuid: {
"total_results": self.search_result_from_cr["message"][
"total-results"
],
"on_page": 1,
"summary": summarized_results,
"results": self.search_result_from_cr["message"]["items"],
}
}
if file_data:
file_data.update(result)
else:
file_data = result
self.app.logger.info("REFINE: Dumping to file")
data_file.truncate(0)
data_file.seek(0)
@@ -96,7 +115,9 @@ class Librarian(object):
async def check_dois_in_local_db(self, doi):
self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
coro = asyncio.to_thread(
search_bot.search_for_doi, doi, self.live_results, self.app.logger
)
result = await coro
return result
@@ -109,25 +130,34 @@ class Librarian(object):
data = response.read()
text = data.decode("utf-8")
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
if re.match(
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
line,
):
return False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
if m := re.match(
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
):
direct_download_link = "https:" + str(m.group(1))
self.app.logger.info (direct_download_link)
self.app.logger.info(direct_download_link)
return True
return False
async def check_if_exists(self, doi, brute_force = False):
async def check_if_exists(self, doi, brute_force=False):
result = {}
result = await self.check_dois_in_local_db(doi)
if brute_force and not result:
for item in doi:
item_link = "https://sci-hub.se/" + item[0]
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
tmp = {
"DOI": item["DOI"],
"exists": await self.check_if_exists_brute_force(item_link),
"data": item,
}
result.update(tmp)
return result
async def refine_query(self, unrefined_reqult, brute_force = False):
async def refine_query(self, unrefined_reqult, brute_force=False):
temp = []
refined_result = []
for key in unrefined_reqult:
@@ -135,7 +165,7 @@ class Librarian(object):
self.logger.info(key)
for item in unrefined_reqult[self.uuid]["summary"]:
if item["container"] == "NO" and item["title"]:
if item["container"] == "NO" and item["title"]:
refined_result.append(item)
for item in refined_result:
@@ -149,20 +179,21 @@ class Librarian(object):
refined_result = temp
self.hit = len(refined_result)
return refined_result
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
async def answer_query(self):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
# Search sql for UUID
# SQL SELECT * FROM RESULTS WHERE UUID = uuid
self.logger.info(f"Search started {self.uuid}")
uuid_found = False
database = None
query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
#ale tymczasowo plik
# tu logika komunikacji z SQL i cała logika związana z wyszukaniem
# ale tymczasowo plik
try:
database = json.load(cr_file)
database = json.load(cr_file)
except JSONDecodeError:
pass
@@ -175,7 +206,7 @@ class Librarian(object):
database.update(answer)
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
json.dump(database, cr_file)
self.result = await self.refine_query(answer)
return self.result
else:
@@ -183,13 +214,14 @@ class Librarian(object):
database = answer
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
json.dump(database, cr_file)
self.result = await self.refine_query(answer)
self.app.logger.info("Refined")
self.done = True
return self.result
#============================= FLASK INTERNALS===============================
# ============================= FLASK INTERNALS===============================
def flask_debug():
"""
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
@@ -198,6 +230,7 @@ def flask_debug():
# trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run():
"""
The `waitress_run` function serves the `app` on host "0.0.0.0"
@@ -205,6 +238,7 @@ def waitress_run():
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
class BackgroundTaskSearch(threading.Thread):
def run(self):
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
@@ -221,7 +255,7 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info("Saving to file")
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
database = json.load(s_file)
except JSONDecodeError:
pass
if database:
@@ -236,17 +270,18 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info(result)
coroutine = asyncio.to_thread(
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
self.app.logger.info("SENT")
_ = await coroutine
self.app.logger.info("SEND CONFIRMED")
time.sleep(1)
#==================================SERVER ROUTES==========================================
# ==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"])
async def query_database():
record = json.loads(request.data)
@@ -254,8 +289,8 @@ async def query_database():
app.logger.info(record["query"])
app.logger.info(record["UUID"])
uuid = record["UUID"]
#TODO: TESTING ONLY======================
uuid = UUID('{12345678-1234-5678-1234-567812345678}')
# TODO: TESTING ONLY======================
uuid = UUID("{12345678-1234-5678-1234-567812345678}")
cl = Librarian(app, record["query"], uuid)
librarian_queue.put(cl)
@@ -267,6 +302,7 @@ async def query_database():
)
return return_data
@app.route("/get_partial_result", methods=["POST"])
async def get_partial():
record = json.loads(request.data)
@@ -282,12 +318,13 @@ async def get_partial():
)
return return_data
#=======================================MAIN===================================================
# =======================================MAIN===================================================
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger = logging.getLogger("conjurer_musician")
logger.setLevel(logging.DEBUG)
threads = []
#threads.append(threading.Thread(target=waitress_run))
# threads.append(threading.Thread(target=waitress_run))
threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch()
bgtask.app = app
@@ -296,7 +333,7 @@ if __name__ == "__main__":
for worker in threads:
try:
logger.info(f"App number: {i}")
i+=1
i += 1
worker.start()
except RuntimeError as e:
logger.error("Exploded")
+6
View File
@@ -0,0 +1,6 @@
#!/bin/bash
cd ./conjurer
git pull
cd ..
cp ./conjurer/file_webservice/conjurer_musician.py ./Conjurer
sudo systemctl restart conjurer_musician.service
+94 -39
View File
@@ -5,19 +5,18 @@ functions for rescanning the music folder, updating the music list, and serving
API endpoints.
"""
import json
import logging
import re
from pathlib import Path
from platform import uname
import threading
import time
from datetime import datetime
from sys import platform
import logging
from flask import Flask, Response, jsonify, request, send_file
from waitress import serve
from logging import handlers
from json.decoder import JSONDecodeError
from pathlib import Path
from platform import uname
from sys import platform
from flask import Flask, jsonify, redirect, request, send_from_directory
from waitress import serve
HOST_ADDRESS = "192.168.1.15"
PORT_ADDRESS = 5000
@@ -30,7 +29,6 @@ if platform in ("linux", "linux2"):
LOGSTORE = "/home/mtuszowski/conjurer/logs/"
ENCODING = "utf-8"
else:
LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
NETRC_FILE = "/home/pi/.netrc"
@@ -40,17 +38,32 @@ if platform in ("linux", "linux2"):
music_file_list = []
def create_playlist():
"""
Reads a JSON file containing a playlist and writes the playlist items to a text file.
The JSON file path is '/home/pi/Conjurer/playlist.json',
and the text file path is '/home/pi/Conjurer/all_playlist.txt'.
Raises:
JSONDecodeError: If the JSON file cannot be decoded.
"""
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
with open("/home/pi/Conjurer/all_playlist.txt", "w", encoding="utf-8") as w_file:
with open(
"/home/pi/Conjurer/all_playlist.txt", "w", encoding="utf-8"
) as w_file:
try:
playlist = json.load(r_file)
for item in playlist:
w_file.write(item)
w_file.write("\n")
except JSONDecodeError:
except json.JSONDecodeError:
pass
def rescan():
"""
The `rescan` function logs a message, scans for mp3 files in a specified folder,
@@ -64,13 +77,12 @@ def rescan():
music_file_list.append(temp_music_file)
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
database = music_file_list
#self.app.logger.info("DUMPING DATA")
# self.app.logger.info("DUMPING DATA")
s_file.truncate(0)
s_file.seek(0)
json.dump(database, s_file)
create_playlist()
#TODO: Zobaczyc czy losowo, itp. lepiej po stronie pythona czy po stronei liquidsoapa. Tak samo modyfikowanie playlisty itp. itd.
#TODO: Ale zaczac od odtwarzania playlisty z tego pliku json (moze np. sie format nie zgadzac albo cos)
def thread_rescan():
"""
@@ -79,15 +91,31 @@ def thread_rescan():
"""
logging.info("Starting filesystemupdater")
while True:
time.sleep(60*60*24)
time.sleep(60 * 60 * 24)
logging.info("Rescan triggered")
rescan()
app = Flask(__name__)
#TODO: Parametry ustalające czy dajemy zwrotkę do bota czy do streamera ? W każdej z funkcji?
#TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
def wyszukaj(word_list, how_many):
logger = logging.getLogger()
# TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
"""
Searches for music files based on a list of keywords and returns a list of matching files.
Args:
word_list (list): A list of keywords to search for.
how_many (int): The maximum number of matching files to return.
Returns:
list: A list of matching files, sorted by their search weight.
"""
if _logger is None:
fun_logger = logging.getLogger()
else:
fun_logger = _logger
return_list = []
search_weight = []
@@ -101,7 +129,7 @@ def wyszukaj(word_list, how_many):
skip_start = 1
for word in word_list[skip_start:]:
token_weight = len(word)
logger.info("Słowo kluczowe: %s", word)
fun_logger.info("Słowo kluczowe: %s", word)
itr = 0
for file in music_file_list:
file = file.split(SEPARATOR_FILE_PATH)
@@ -141,7 +169,7 @@ def wyszukaj(word_list, how_many):
else:
skip = 4
matched_times = 1
#TODO: To do wątków albo for asynchroniczny
# TODO: To do wątków albo for asynchroniczny
for itm in all_words[skip:]:
pingu += 1
pattern = ".*" + word + ".*"
@@ -157,26 +185,40 @@ def wyszukaj(word_list, how_many):
matched_times += 1
itr += 1
logger.info("Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start)
fun_logger.info(
"Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start
)
time_start = datetime.now()
item_to_search = max_weight(search_weight)
itr = 0
if item_to_search == 0:
return None
return []
not_found = True
return_list = []
if int(how_many) <= 0:
logger.info("Jeden plik do zagrania")
fun_logger.info("Jeden plik do zagrania")
while not_found:
if search_weight[itr][0] == item_to_search:
return_list.append(search_weight[itr])
return return_list
if not return_to_bot:
with open(
"/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
) as s_file:
s_file.write(search_weight[itr][1])
break
itr += 1
else:
logger.info("Wiele plików do zagrania")
fun_logger.info("Wiele plików do zagrania")
search_weight.sort(key=lambda x: x[0], reverse=True)
return_list.extend(search_weight[: int(how_many)])
return return_list
if not return_to_bot:
for item in return_list:
with open(
"/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
) as s_file:
s_file.write(item[1])
return return_list
def max_weight(lista):
"""
@@ -205,17 +247,25 @@ def remove_characters(string, character):
"""
return string.replace(character, "")
@app.route("/stream", methods=["GET"])
def stream_music():
def stream_music_hls():
"""
The function `get_music_list` returns a JSON object containing a list of music files.
:return: A JSON response containing a key "music_file_list" with the value of the variable
`music_file_list`.
"""
m3u8 = Path("/tmp/hls/stream.m3u8")
r = Response(m3u8)
r.headers["Content-Type"] = "application/mpegURL"
return r, 200
return send_from_directory("/tmp/hls", "stream.m3u8")
@app.route("/stream_mp3", methods=["GET"])
def stream_music_mp3():
"""
The function `get_music_list` returns a JSON object containing a list of music files.
:return: A JSON response containing a key "music_file_list" with the value of the variable
`music_file_list`.
"""
return redirect("http://www.example.com", code=302)
@app.route("/mp3", methods=["GET"])
@@ -248,6 +298,7 @@ def update_music_list():
)
return return_data
@app.route("/get_music", methods=["POST"])
def look_for_playlist():
"""
@@ -264,7 +315,7 @@ def look_for_playlist():
app.logger.info(record["lista_slow"])
app.logger.info(record["UUID"])
app.logger.info(record["dlugosc_plejlisty"])
return_data = wyszukaj(record["lista_slow"],record["dlugosc_plejlisty"])
return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger)
return_data = (
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
@@ -281,6 +332,7 @@ def flask_debug():
# trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run():
"""
The `waitress_run` function serves the `app` on host "0.0.0.0"
@@ -288,16 +340,19 @@ def waitress_run():
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger = logging.getLogger("conjurer_musician")
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
handler = handlers.RotatingFileHandler(
filename=LOGFILE,
encoding=ENCODING,
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
filename=LOGFILE,
encoding=ENCODING,
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
)
handler.setFormatter(formatter)
logger.addHandler(handler)
+1 -1
View File
@@ -4,7 +4,7 @@ After=multi-user.target
[Service]
Type=simple
ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/conjurer_musician.py
Restart=on-abort
[Install]
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Radio_conjurer
After=multi-user.target
[Service]
Type=simple
ExecStart=/home/pi/Conjurer/ liquidsoap /home/pi/Conjurer/radio_conjurer.liq
Restart=on-abort
[Install]
WantedBy=multi-user.target
+6 -2
View File
@@ -1,5 +1,9 @@
#!/bin/bash
cd /home/pi
mdkir Conjurer
cd Conjurer
python3 -m venv /home/pi/Conjurer/env
cp /home/pi/conjurer/requirements_bot.txt /home/pi/Conjurer/
source ./env/bin/activate
./env/bin/python3 -m pip install --upgrade pip
./env/bin/python3 -m pip install -r requirements_bot.txt
@@ -8,5 +12,5 @@ sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.
deactivate
sudo cp ./conjurer.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl start conjurer_musician.service
sudo systemctl enable conjurer_musician.service
sudo systemctl start conjurer.service
sudo systemctl enable conjurer.service
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
cd /home/pi
mdkir Conjurer
cd Conjurer
python3 -m venv /home/pi/Conjurer/env
source ./env/bin/activate
cp /home/pi/conjurer/conjurer_musician/requirements_bot.txt /home/pi/Conjurer/
cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
./env/bin/python3 -m pip install --upgrade pip
./env/bin/python3 -m pip install -r requirements_bot.txt
deactivate
sudo cp ./file_webservice/conjurer_musician.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl start conjurer_musician.service
sudo systemctl enable conjurer_musician.service