mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
Fixing the bug in the conjurer_musician.py file.
Fx Fix fx Fix ? Revert "Fix" This reverts commit 4e185f7b34dfede04ed36ab9813bd9d1f7fb8b35. Adding deploy for music bot
This commit is contained in:
@@ -1,27 +1,29 @@
|
|||||||
import re
|
import asyncio
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import netrc
|
||||||
|
import re
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from flask import Flask, jsonify, request
|
|
||||||
import netrc
|
|
||||||
import json
|
|
||||||
import asyncio
|
|
||||||
from json.decoder import JSONDecodeError
|
from json.decoder import JSONDecodeError
|
||||||
|
from queue import Queue
|
||||||
|
from typing import Callable
|
||||||
from urllib.request import urlopen
|
from urllib.request import urlopen
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import search_bot
|
||||||
|
from flask import Flask, jsonify, request
|
||||||
from habanero import Crossref
|
from habanero import Crossref
|
||||||
from waitress import serve
|
from waitress import serve
|
||||||
from queue import Queue
|
|
||||||
import search_bot
|
|
||||||
import requests
|
|
||||||
from typing import Callable
|
|
||||||
from uuid import UUID
|
|
||||||
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
||||||
|
|
||||||
HOST_ADDRESS = "192.168.1.192"
|
HOST_ADDRESS = "192.168.1.192"
|
||||||
PORT_ADDRESS = 5001
|
PORT_ADDRESS = 5001
|
||||||
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
|
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
|
||||||
SEND_RESULTS = "/conjurer"
|
SEND_RESULTS = "/conjurer"
|
||||||
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
|
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
|
||||||
|
|
||||||
ENCODING = "utf-8"
|
ENCODING = "utf-8"
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
@@ -29,6 +31,7 @@ app = Flask(__name__)
|
|||||||
librarian_queue = Queue()
|
librarian_queue = Queue()
|
||||||
librarian_list = []
|
librarian_list = []
|
||||||
|
|
||||||
|
|
||||||
class Librarian(object):
|
class Librarian(object):
|
||||||
|
|
||||||
def __init__(self, app, query, uuid) -> None:
|
def __init__(self, app, query, uuid) -> None:
|
||||||
@@ -49,25 +52,25 @@ class Librarian(object):
|
|||||||
|
|
||||||
async def search_crossref(self, query):
|
async def search_crossref(self, query):
|
||||||
self.app.logger.info("STARTED SEARCH")
|
self.app.logger.info("STARTED SEARCH")
|
||||||
result = self.cr.works(query = query, limit = 1000)
|
result = self.cr.works(query=query, limit=1000)
|
||||||
self.search_result_from_cr.update(result)
|
self.search_result_from_cr.update(result)
|
||||||
self.total = result['message']['total-results']
|
self.total = result["message"]["total-results"]
|
||||||
self.fetched += len(result['message']['items'])
|
self.fetched += len(result["message"]["items"])
|
||||||
self.logger.info (self.total)
|
self.logger.info(self.total)
|
||||||
self.logger.info(self.fetched)
|
self.logger.info(self.fetched)
|
||||||
while self.total > self.fetched and self.limit > self.fetched:
|
while self.total > self.fetched and self.limit > self.fetched:
|
||||||
result = self.cr.works(query = query, limit = 1000)
|
result = self.cr.works(query=query, limit=1000)
|
||||||
self.search_result_from_cr.update(result)
|
self.search_result_from_cr.update(result)
|
||||||
self.total = result['message']['total-results']
|
self.total = result["message"]["total-results"]
|
||||||
self.fetched += len(result['message']['items'])
|
self.fetched += len(result["message"]["items"])
|
||||||
self.logger.info (self.total)
|
self.logger.info(self.total)
|
||||||
self.logger.info(self.fetched)
|
self.logger.info(self.fetched)
|
||||||
|
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
self.app.logger.info("CROSSREF DONE")
|
self.app.logger.info("CROSSREF DONE")
|
||||||
#TODO: add to sql with UUID
|
# TODO: add to sql with UUID
|
||||||
#sql row - title, doi, author, uuid, last retrieved, is_available
|
# sql row - title, doi, author, uuid, last retrieved, is_available
|
||||||
|
|
||||||
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
|
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
|
||||||
# First we load existing data into a dict.
|
# First we load existing data into a dict.
|
||||||
try:
|
try:
|
||||||
@@ -76,18 +79,34 @@ class Librarian(object):
|
|||||||
file_data = {}
|
file_data = {}
|
||||||
summarized_results = []
|
summarized_results = []
|
||||||
self.app.logger.info("REFINE: Removing all derived works from the list")
|
self.app.logger.info("REFINE: Removing all derived works from the list")
|
||||||
for item in self.search_result_from_cr['message']['items']:
|
for item in self.search_result_from_cr["message"]["items"]:
|
||||||
container = "NO"
|
container = "NO"
|
||||||
if "container-title" in item:
|
if "container-title" in item:
|
||||||
container = "YES"
|
container = "YES"
|
||||||
else:
|
else:
|
||||||
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
|
summarized_results.append(
|
||||||
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
|
{
|
||||||
|
"DOI": item["DOI"],
|
||||||
|
"title": item["title"] if "title" in item else None,
|
||||||
|
"type": item["type"] if "type" in item else None,
|
||||||
|
"container": container,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
self.uuid: {
|
||||||
|
"total_results": self.search_result_from_cr["message"][
|
||||||
|
"total-results"
|
||||||
|
],
|
||||||
|
"on_page": 1,
|
||||||
|
"summary": summarized_results,
|
||||||
|
"results": self.search_result_from_cr["message"]["items"],
|
||||||
|
}
|
||||||
|
}
|
||||||
if file_data:
|
if file_data:
|
||||||
file_data.update(result)
|
file_data.update(result)
|
||||||
else:
|
else:
|
||||||
file_data = result
|
file_data = result
|
||||||
|
|
||||||
self.app.logger.info("REFINE: Dumping to file")
|
self.app.logger.info("REFINE: Dumping to file")
|
||||||
data_file.truncate(0)
|
data_file.truncate(0)
|
||||||
data_file.seek(0)
|
data_file.seek(0)
|
||||||
@@ -96,7 +115,9 @@ class Librarian(object):
|
|||||||
|
|
||||||
async def check_dois_in_local_db(self, doi):
|
async def check_dois_in_local_db(self, doi):
|
||||||
self.app.logger.info("REFINE: Running search in the backend app")
|
self.app.logger.info("REFINE: Running search in the backend app")
|
||||||
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
|
coro = asyncio.to_thread(
|
||||||
|
search_bot.search_for_doi, doi, self.live_results, self.app.logger
|
||||||
|
)
|
||||||
result = await coro
|
result = await coro
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@@ -109,25 +130,34 @@ class Librarian(object):
|
|||||||
data = response.read()
|
data = response.read()
|
||||||
text = data.decode("utf-8")
|
text = data.decode("utf-8")
|
||||||
for line in text.splitlines():
|
for line in text.splitlines():
|
||||||
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
|
if re.match(
|
||||||
|
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
|
||||||
|
line,
|
||||||
|
):
|
||||||
return False
|
return False
|
||||||
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
|
if m := re.match(
|
||||||
|
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
|
||||||
|
):
|
||||||
direct_download_link = "https:" + str(m.group(1))
|
direct_download_link = "https:" + str(m.group(1))
|
||||||
self.app.logger.info (direct_download_link)
|
self.app.logger.info(direct_download_link)
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
async def check_if_exists(self, doi, brute_force = False):
|
async def check_if_exists(self, doi, brute_force=False):
|
||||||
result = {}
|
result = {}
|
||||||
result = await self.check_dois_in_local_db(doi)
|
result = await self.check_dois_in_local_db(doi)
|
||||||
if brute_force and not result:
|
if brute_force and not result:
|
||||||
for item in doi:
|
for item in doi:
|
||||||
item_link = "https://sci-hub.se/" + item[0]
|
item_link = "https://sci-hub.se/" + item[0]
|
||||||
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
|
tmp = {
|
||||||
|
"DOI": item["DOI"],
|
||||||
|
"exists": await self.check_if_exists_brute_force(item_link),
|
||||||
|
"data": item,
|
||||||
|
}
|
||||||
result.update(tmp)
|
result.update(tmp)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def refine_query(self, unrefined_reqult, brute_force = False):
|
async def refine_query(self, unrefined_reqult, brute_force=False):
|
||||||
temp = []
|
temp = []
|
||||||
refined_result = []
|
refined_result = []
|
||||||
for key in unrefined_reqult:
|
for key in unrefined_reqult:
|
||||||
@@ -135,7 +165,7 @@ class Librarian(object):
|
|||||||
self.logger.info(key)
|
self.logger.info(key)
|
||||||
|
|
||||||
for item in unrefined_reqult[self.uuid]["summary"]:
|
for item in unrefined_reqult[self.uuid]["summary"]:
|
||||||
if item["container"] == "NO" and item["title"]:
|
if item["container"] == "NO" and item["title"]:
|
||||||
refined_result.append(item)
|
refined_result.append(item)
|
||||||
|
|
||||||
for item in refined_result:
|
for item in refined_result:
|
||||||
@@ -149,20 +179,21 @@ class Librarian(object):
|
|||||||
refined_result = temp
|
refined_result = temp
|
||||||
self.hit = len(refined_result)
|
self.hit = len(refined_result)
|
||||||
return refined_result
|
return refined_result
|
||||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
|
|
||||||
|
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
|
||||||
async def answer_query(self):
|
async def answer_query(self):
|
||||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
|
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
|
||||||
#Search sql for UUID
|
# Search sql for UUID
|
||||||
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
|
# SQL SELECT * FROM RESULTS WHERE UUID = uuid
|
||||||
self.logger.info(f"Search started {self.uuid}")
|
self.logger.info(f"Search started {self.uuid}")
|
||||||
uuid_found = False
|
uuid_found = False
|
||||||
database = None
|
database = None
|
||||||
query = self.query
|
query = self.query
|
||||||
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
|
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
|
||||||
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
|
# tu logika komunikacji z SQL i cała logika związana z wyszukaniem
|
||||||
#ale tymczasowo plik
|
# ale tymczasowo plik
|
||||||
try:
|
try:
|
||||||
database = json.load(cr_file)
|
database = json.load(cr_file)
|
||||||
except JSONDecodeError:
|
except JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -175,7 +206,7 @@ class Librarian(object):
|
|||||||
database.update(answer)
|
database.update(answer)
|
||||||
cr_file.truncate(0)
|
cr_file.truncate(0)
|
||||||
cr_file.seek(0)
|
cr_file.seek(0)
|
||||||
json.dump(database,cr_file)
|
json.dump(database, cr_file)
|
||||||
self.result = await self.refine_query(answer)
|
self.result = await self.refine_query(answer)
|
||||||
return self.result
|
return self.result
|
||||||
else:
|
else:
|
||||||
@@ -183,13 +214,14 @@ class Librarian(object):
|
|||||||
database = answer
|
database = answer
|
||||||
cr_file.truncate(0)
|
cr_file.truncate(0)
|
||||||
cr_file.seek(0)
|
cr_file.seek(0)
|
||||||
json.dump(database,cr_file)
|
json.dump(database, cr_file)
|
||||||
self.result = await self.refine_query(answer)
|
self.result = await self.refine_query(answer)
|
||||||
self.app.logger.info("Refined")
|
self.app.logger.info("Refined")
|
||||||
self.done = True
|
self.done = True
|
||||||
return self.result
|
return self.result
|
||||||
|
|
||||||
#============================= FLASK INTERNALS===============================
|
|
||||||
|
# ============================= FLASK INTERNALS===============================
|
||||||
def flask_debug():
|
def flask_debug():
|
||||||
"""
|
"""
|
||||||
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
|
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
|
||||||
@@ -198,6 +230,7 @@ def flask_debug():
|
|||||||
# trunk-ignore(bandit/B201)
|
# trunk-ignore(bandit/B201)
|
||||||
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
def waitress_run():
|
def waitress_run():
|
||||||
"""
|
"""
|
||||||
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
||||||
@@ -205,6 +238,7 @@ def waitress_run():
|
|||||||
"""
|
"""
|
||||||
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
class BackgroundTaskSearch(threading.Thread):
|
class BackgroundTaskSearch(threading.Thread):
|
||||||
def run(self):
|
def run(self):
|
||||||
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
|
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
|
||||||
@@ -221,7 +255,7 @@ class BackgroundTaskSearch(threading.Thread):
|
|||||||
self.app.logger.info("Saving to file")
|
self.app.logger.info("Saving to file")
|
||||||
with open("s_results.json", "r+", encoding="utf-8") as s_file:
|
with open("s_results.json", "r+", encoding="utf-8") as s_file:
|
||||||
try:
|
try:
|
||||||
database = json.load(s_file)
|
database = json.load(s_file)
|
||||||
except JSONDecodeError:
|
except JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
if database:
|
if database:
|
||||||
@@ -236,17 +270,18 @@ class BackgroundTaskSearch(threading.Thread):
|
|||||||
|
|
||||||
self.app.logger.info(result)
|
self.app.logger.info(result)
|
||||||
coroutine = asyncio.to_thread(
|
coroutine = asyncio.to_thread(
|
||||||
requests.post,
|
requests.post,
|
||||||
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
|
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
|
||||||
json=result,
|
json=result,
|
||||||
timeout=360,
|
timeout=360,
|
||||||
)
|
)
|
||||||
self.app.logger.info("SENT")
|
self.app.logger.info("SENT")
|
||||||
_ = await coroutine
|
_ = await coroutine
|
||||||
self.app.logger.info("SEND CONFIRMED")
|
self.app.logger.info("SEND CONFIRMED")
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
#==================================SERVER ROUTES==========================================
|
|
||||||
|
# ==================================SERVER ROUTES==========================================
|
||||||
@app.route("/query", methods=["POST"])
|
@app.route("/query", methods=["POST"])
|
||||||
async def query_database():
|
async def query_database():
|
||||||
record = json.loads(request.data)
|
record = json.loads(request.data)
|
||||||
@@ -254,8 +289,8 @@ async def query_database():
|
|||||||
app.logger.info(record["query"])
|
app.logger.info(record["query"])
|
||||||
app.logger.info(record["UUID"])
|
app.logger.info(record["UUID"])
|
||||||
uuid = record["UUID"]
|
uuid = record["UUID"]
|
||||||
#TODO: TESTING ONLY======================
|
# TODO: TESTING ONLY======================
|
||||||
uuid = UUID('{12345678-1234-5678-1234-567812345678}')
|
uuid = UUID("{12345678-1234-5678-1234-567812345678}")
|
||||||
|
|
||||||
cl = Librarian(app, record["query"], uuid)
|
cl = Librarian(app, record["query"], uuid)
|
||||||
librarian_queue.put(cl)
|
librarian_queue.put(cl)
|
||||||
@@ -267,6 +302,7 @@ async def query_database():
|
|||||||
)
|
)
|
||||||
return return_data
|
return return_data
|
||||||
|
|
||||||
|
|
||||||
@app.route("/get_partial_result", methods=["POST"])
|
@app.route("/get_partial_result", methods=["POST"])
|
||||||
async def get_partial():
|
async def get_partial():
|
||||||
record = json.loads(request.data)
|
record = json.loads(request.data)
|
||||||
@@ -282,12 +318,13 @@ async def get_partial():
|
|||||||
)
|
)
|
||||||
return return_data
|
return return_data
|
||||||
|
|
||||||
#=======================================MAIN===================================================
|
|
||||||
|
# =======================================MAIN===================================================
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logger = logging.getLogger('conjurer_musician')
|
logger = logging.getLogger("conjurer_musician")
|
||||||
logger.setLevel(logging.DEBUG)
|
logger.setLevel(logging.DEBUG)
|
||||||
threads = []
|
threads = []
|
||||||
#threads.append(threading.Thread(target=waitress_run))
|
# threads.append(threading.Thread(target=waitress_run))
|
||||||
threads.append(threading.Thread(target=flask_debug))
|
threads.append(threading.Thread(target=flask_debug))
|
||||||
bgtask = BackgroundTaskSearch()
|
bgtask = BackgroundTaskSearch()
|
||||||
bgtask.app = app
|
bgtask.app = app
|
||||||
@@ -296,7 +333,7 @@ if __name__ == "__main__":
|
|||||||
for worker in threads:
|
for worker in threads:
|
||||||
try:
|
try:
|
||||||
logger.info(f"App number: {i}")
|
logger.info(f"App number: {i}")
|
||||||
i+=1
|
i += 1
|
||||||
worker.start()
|
worker.start()
|
||||||
except RuntimeError as e:
|
except RuntimeError as e:
|
||||||
logger.error("Exploded")
|
logger.error("Exploded")
|
||||||
|
|||||||
Executable
+6
@@ -0,0 +1,6 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
cd ./conjurer
|
||||||
|
git pull
|
||||||
|
cd ..
|
||||||
|
cp ./conjurer/file_webservice/conjurer_musician.py ./Conjurer
|
||||||
|
sudo systemctl restart conjurer_musician.service
|
||||||
@@ -5,19 +5,18 @@ functions for rescanning the music folder, updating the music list, and serving
|
|||||||
API endpoints.
|
API endpoints.
|
||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
from pathlib import Path
|
|
||||||
from platform import uname
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from sys import platform
|
|
||||||
import logging
|
|
||||||
from flask import Flask, Response, jsonify, request, send_file
|
|
||||||
from waitress import serve
|
|
||||||
from logging import handlers
|
from logging import handlers
|
||||||
from json.decoder import JSONDecodeError
|
from pathlib import Path
|
||||||
|
from platform import uname
|
||||||
|
from sys import platform
|
||||||
|
|
||||||
|
from flask import Flask, jsonify, redirect, request, send_from_directory
|
||||||
|
from waitress import serve
|
||||||
|
|
||||||
HOST_ADDRESS = "192.168.1.15"
|
HOST_ADDRESS = "192.168.1.15"
|
||||||
PORT_ADDRESS = 5000
|
PORT_ADDRESS = 5000
|
||||||
@@ -30,7 +29,6 @@ if platform in ("linux", "linux2"):
|
|||||||
LOGSTORE = "/home/mtuszowski/conjurer/logs/"
|
LOGSTORE = "/home/mtuszowski/conjurer/logs/"
|
||||||
ENCODING = "utf-8"
|
ENCODING = "utf-8"
|
||||||
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
|
LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
|
||||||
NETRC_FILE = "/home/pi/.netrc"
|
NETRC_FILE = "/home/pi/.netrc"
|
||||||
@@ -40,17 +38,32 @@ if platform in ("linux", "linux2"):
|
|||||||
|
|
||||||
|
|
||||||
music_file_list = []
|
music_file_list = []
|
||||||
|
|
||||||
|
|
||||||
def create_playlist():
|
def create_playlist():
|
||||||
|
"""
|
||||||
|
Reads a JSON file containing a playlist and writes the playlist items to a text file.
|
||||||
|
|
||||||
|
The JSON file path is '/home/pi/Conjurer/playlist.json',
|
||||||
|
and the text file path is '/home/pi/Conjurer/all_playlist.txt'.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
JSONDecodeError: If the JSON file cannot be decoded.
|
||||||
|
|
||||||
|
"""
|
||||||
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
|
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
|
||||||
with open("/home/pi/Conjurer/all_playlist.txt", "w", encoding="utf-8") as w_file:
|
with open(
|
||||||
|
"/home/pi/Conjurer/all_playlist.txt", "w", encoding="utf-8"
|
||||||
|
) as w_file:
|
||||||
try:
|
try:
|
||||||
playlist = json.load(r_file)
|
playlist = json.load(r_file)
|
||||||
for item in playlist:
|
for item in playlist:
|
||||||
w_file.write(item)
|
w_file.write(item)
|
||||||
w_file.write("\n")
|
w_file.write("\n")
|
||||||
except JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def rescan():
|
def rescan():
|
||||||
"""
|
"""
|
||||||
The `rescan` function logs a message, scans for mp3 files in a specified folder,
|
The `rescan` function logs a message, scans for mp3 files in a specified folder,
|
||||||
@@ -64,13 +77,12 @@ def rescan():
|
|||||||
music_file_list.append(temp_music_file)
|
music_file_list.append(temp_music_file)
|
||||||
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
|
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
|
||||||
database = music_file_list
|
database = music_file_list
|
||||||
#self.app.logger.info("DUMPING DATA")
|
# self.app.logger.info("DUMPING DATA")
|
||||||
s_file.truncate(0)
|
s_file.truncate(0)
|
||||||
s_file.seek(0)
|
s_file.seek(0)
|
||||||
json.dump(database, s_file)
|
json.dump(database, s_file)
|
||||||
create_playlist()
|
create_playlist()
|
||||||
#TODO: Zobaczyc czy losowo, itp. lepiej po stronie pythona czy po stronei liquidsoapa. Tak samo modyfikowanie playlisty itp. itd.
|
|
||||||
#TODO: Ale zaczac od odtwarzania playlisty z tego pliku json (moze np. sie format nie zgadzac albo cos)
|
|
||||||
|
|
||||||
def thread_rescan():
|
def thread_rescan():
|
||||||
"""
|
"""
|
||||||
@@ -79,15 +91,31 @@ def thread_rescan():
|
|||||||
"""
|
"""
|
||||||
logging.info("Starting filesystemupdater")
|
logging.info("Starting filesystemupdater")
|
||||||
while True:
|
while True:
|
||||||
time.sleep(60*60*24)
|
time.sleep(60 * 60 * 24)
|
||||||
logging.info("Rescan triggered")
|
logging.info("Rescan triggered")
|
||||||
rescan()
|
rescan()
|
||||||
|
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
#TODO: Parametry ustalające czy dajemy zwrotkę do bota czy do streamera ? W każdej z funkcji?
|
|
||||||
#TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
|
|
||||||
def wyszukaj(word_list, how_many):
|
# TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
|
||||||
logger = logging.getLogger()
|
def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
|
||||||
|
"""
|
||||||
|
Searches for music files based on a list of keywords and returns a list of matching files.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
word_list (list): A list of keywords to search for.
|
||||||
|
how_many (int): The maximum number of matching files to return.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list: A list of matching files, sorted by their search weight.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if _logger is None:
|
||||||
|
fun_logger = logging.getLogger()
|
||||||
|
else:
|
||||||
|
fun_logger = _logger
|
||||||
return_list = []
|
return_list = []
|
||||||
search_weight = []
|
search_weight = []
|
||||||
|
|
||||||
@@ -101,7 +129,7 @@ def wyszukaj(word_list, how_many):
|
|||||||
skip_start = 1
|
skip_start = 1
|
||||||
for word in word_list[skip_start:]:
|
for word in word_list[skip_start:]:
|
||||||
token_weight = len(word)
|
token_weight = len(word)
|
||||||
logger.info("Słowo kluczowe: %s", word)
|
fun_logger.info("Słowo kluczowe: %s", word)
|
||||||
itr = 0
|
itr = 0
|
||||||
for file in music_file_list:
|
for file in music_file_list:
|
||||||
file = file.split(SEPARATOR_FILE_PATH)
|
file = file.split(SEPARATOR_FILE_PATH)
|
||||||
@@ -141,7 +169,7 @@ def wyszukaj(word_list, how_many):
|
|||||||
else:
|
else:
|
||||||
skip = 4
|
skip = 4
|
||||||
matched_times = 1
|
matched_times = 1
|
||||||
#TODO: To do wątków albo for asynchroniczny
|
# TODO: To do wątków albo for asynchroniczny
|
||||||
for itm in all_words[skip:]:
|
for itm in all_words[skip:]:
|
||||||
pingu += 1
|
pingu += 1
|
||||||
pattern = ".*" + word + ".*"
|
pattern = ".*" + word + ".*"
|
||||||
@@ -157,26 +185,40 @@ def wyszukaj(word_list, how_many):
|
|||||||
matched_times += 1
|
matched_times += 1
|
||||||
itr += 1
|
itr += 1
|
||||||
|
|
||||||
logger.info("Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start)
|
fun_logger.info(
|
||||||
|
"Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start
|
||||||
|
)
|
||||||
time_start = datetime.now()
|
time_start = datetime.now()
|
||||||
item_to_search = max_weight(search_weight)
|
item_to_search = max_weight(search_weight)
|
||||||
itr = 0
|
itr = 0
|
||||||
if item_to_search == 0:
|
if item_to_search == 0:
|
||||||
return None
|
return []
|
||||||
not_found = True
|
not_found = True
|
||||||
return_list = []
|
return_list = []
|
||||||
if int(how_many) <= 0:
|
if int(how_many) <= 0:
|
||||||
logger.info("Jeden plik do zagrania")
|
fun_logger.info("Jeden plik do zagrania")
|
||||||
while not_found:
|
while not_found:
|
||||||
if search_weight[itr][0] == item_to_search:
|
if search_weight[itr][0] == item_to_search:
|
||||||
return_list.append(search_weight[itr])
|
return_list.append(search_weight[itr])
|
||||||
return return_list
|
if not return_to_bot:
|
||||||
|
with open(
|
||||||
|
"/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
|
||||||
|
) as s_file:
|
||||||
|
s_file.write(search_weight[itr][1])
|
||||||
|
break
|
||||||
itr += 1
|
itr += 1
|
||||||
else:
|
else:
|
||||||
logger.info("Wiele plików do zagrania")
|
fun_logger.info("Wiele plików do zagrania")
|
||||||
search_weight.sort(key=lambda x: x[0], reverse=True)
|
search_weight.sort(key=lambda x: x[0], reverse=True)
|
||||||
return_list.extend(search_weight[: int(how_many)])
|
return_list.extend(search_weight[: int(how_many)])
|
||||||
return return_list
|
if not return_to_bot:
|
||||||
|
for item in return_list:
|
||||||
|
with open(
|
||||||
|
"/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
|
||||||
|
) as s_file:
|
||||||
|
s_file.write(item[1])
|
||||||
|
return return_list
|
||||||
|
|
||||||
|
|
||||||
def max_weight(lista):
|
def max_weight(lista):
|
||||||
"""
|
"""
|
||||||
@@ -205,17 +247,25 @@ def remove_characters(string, character):
|
|||||||
"""
|
"""
|
||||||
return string.replace(character, "")
|
return string.replace(character, "")
|
||||||
|
|
||||||
|
|
||||||
@app.route("/stream", methods=["GET"])
|
@app.route("/stream", methods=["GET"])
|
||||||
def stream_music():
|
def stream_music_hls():
|
||||||
"""
|
"""
|
||||||
The function `get_music_list` returns a JSON object containing a list of music files.
|
The function `get_music_list` returns a JSON object containing a list of music files.
|
||||||
:return: A JSON response containing a key "music_file_list" with the value of the variable
|
:return: A JSON response containing a key "music_file_list" with the value of the variable
|
||||||
`music_file_list`.
|
`music_file_list`.
|
||||||
"""
|
"""
|
||||||
m3u8 = Path("/tmp/hls/stream.m3u8")
|
return send_from_directory("/tmp/hls", "stream.m3u8")
|
||||||
r = Response(m3u8)
|
|
||||||
r.headers["Content-Type"] = "application/mpegURL"
|
|
||||||
return r, 200
|
@app.route("/stream_mp3", methods=["GET"])
|
||||||
|
def stream_music_mp3():
|
||||||
|
"""
|
||||||
|
The function `get_music_list` returns a JSON object containing a list of music files.
|
||||||
|
:return: A JSON response containing a key "music_file_list" with the value of the variable
|
||||||
|
`music_file_list`.
|
||||||
|
"""
|
||||||
|
return redirect("http://www.example.com", code=302)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/mp3", methods=["GET"])
|
@app.route("/mp3", methods=["GET"])
|
||||||
@@ -248,6 +298,7 @@ def update_music_list():
|
|||||||
)
|
)
|
||||||
return return_data
|
return return_data
|
||||||
|
|
||||||
|
|
||||||
@app.route("/get_music", methods=["POST"])
|
@app.route("/get_music", methods=["POST"])
|
||||||
def look_for_playlist():
|
def look_for_playlist():
|
||||||
"""
|
"""
|
||||||
@@ -264,7 +315,7 @@ def look_for_playlist():
|
|||||||
app.logger.info(record["lista_slow"])
|
app.logger.info(record["lista_slow"])
|
||||||
app.logger.info(record["UUID"])
|
app.logger.info(record["UUID"])
|
||||||
app.logger.info(record["dlugosc_plejlisty"])
|
app.logger.info(record["dlugosc_plejlisty"])
|
||||||
return_data = wyszukaj(record["lista_slow"],record["dlugosc_plejlisty"])
|
return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger)
|
||||||
|
|
||||||
return_data = (
|
return_data = (
|
||||||
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
|
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
|
||||||
@@ -281,6 +332,7 @@ def flask_debug():
|
|||||||
# trunk-ignore(bandit/B201)
|
# trunk-ignore(bandit/B201)
|
||||||
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
def waitress_run():
|
def waitress_run():
|
||||||
"""
|
"""
|
||||||
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
||||||
@@ -288,16 +340,19 @@ def waitress_run():
|
|||||||
"""
|
"""
|
||||||
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logger = logging.getLogger('conjurer_musician')
|
logger = logging.getLogger("conjurer_musician")
|
||||||
logger.setLevel(logging.DEBUG)
|
logger.setLevel(logging.DEBUG)
|
||||||
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
formatter = logging.Formatter(
|
||||||
|
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||||
|
)
|
||||||
handler = handlers.RotatingFileHandler(
|
handler = handlers.RotatingFileHandler(
|
||||||
filename=LOGFILE,
|
filename=LOGFILE,
|
||||||
encoding=ENCODING,
|
encoding=ENCODING,
|
||||||
mode="a",
|
mode="a",
|
||||||
maxBytes=6 * 1024 * 1024,
|
maxBytes=6 * 1024 * 1024,
|
||||||
backupCount=6,
|
backupCount=6,
|
||||||
)
|
)
|
||||||
handler.setFormatter(formatter)
|
handler.setFormatter(formatter)
|
||||||
logger.addHandler(handler)
|
logger.addHandler(handler)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ After=multi-user.target
|
|||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
|
ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/conjurer_musician.py
|
||||||
Restart=on-abort
|
Restart=on-abort
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Radio_conjurer
|
||||||
|
After=multi-user.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/home/pi/Conjurer/ liquidsoap /home/pi/Conjurer/radio_conjurer.liq
|
||||||
|
Restart=on-abort
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
+6
-2
@@ -1,5 +1,9 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
cd /home/pi
|
||||||
|
mdkir Conjurer
|
||||||
|
cd Conjurer
|
||||||
python3 -m venv /home/pi/Conjurer/env
|
python3 -m venv /home/pi/Conjurer/env
|
||||||
|
cp /home/pi/conjurer/requirements_bot.txt /home/pi/Conjurer/
|
||||||
source ./env/bin/activate
|
source ./env/bin/activate
|
||||||
./env/bin/python3 -m pip install --upgrade pip
|
./env/bin/python3 -m pip install --upgrade pip
|
||||||
./env/bin/python3 -m pip install -r requirements_bot.txt
|
./env/bin/python3 -m pip install -r requirements_bot.txt
|
||||||
@@ -8,5 +12,5 @@ sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.
|
|||||||
deactivate
|
deactivate
|
||||||
sudo cp ./conjurer.service /etc/systemd/system/
|
sudo cp ./conjurer.service /etc/systemd/system/
|
||||||
sudo systemctl daemon-reload
|
sudo systemctl daemon-reload
|
||||||
sudo systemctl start conjurer_musician.service
|
sudo systemctl start conjurer.service
|
||||||
sudo systemctl enable conjurer_musician.service
|
sudo systemctl enable conjurer.service
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
cd /home/pi
|
||||||
|
mdkir Conjurer
|
||||||
|
cd Conjurer
|
||||||
|
python3 -m venv /home/pi/Conjurer/env
|
||||||
|
source ./env/bin/activate
|
||||||
|
cp /home/pi/conjurer/conjurer_musician/requirements_bot.txt /home/pi/Conjurer/
|
||||||
|
cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
|
||||||
|
cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
|
||||||
|
./env/bin/python3 -m pip install --upgrade pip
|
||||||
|
./env/bin/python3 -m pip install -r requirements_bot.txt
|
||||||
|
deactivate
|
||||||
|
sudo cp ./file_webservice/conjurer_musician.service /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl start conjurer_musician.service
|
||||||
|
sudo systemctl enable conjurer_musician.service
|
||||||
Reference in New Issue
Block a user