mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-15 14:22:13 +00:00
Compare commits
6 Commits
1.6
...
1st_radio_pr
| Author | SHA1 | Date | |
|---|---|---|---|
| 30d0207368 | |||
| d27b477db8 | |||
| ca6896ebcf | |||
| 0375b0a892 | |||
| 1caecceb10 | |||
| 5cb51fb296 |
@@ -1,27 +1,29 @@
|
|||||||
import re
|
import asyncio
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import netrc
|
||||||
|
import re
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from flask import Flask, jsonify, request
|
|
||||||
import netrc
|
|
||||||
import json
|
|
||||||
import asyncio
|
|
||||||
from json.decoder import JSONDecodeError
|
from json.decoder import JSONDecodeError
|
||||||
|
from queue import Queue
|
||||||
|
from typing import Callable
|
||||||
from urllib.request import urlopen
|
from urllib.request import urlopen
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import search_bot
|
||||||
|
from flask import Flask, jsonify, request
|
||||||
from habanero import Crossref
|
from habanero import Crossref
|
||||||
from waitress import serve
|
from waitress import serve
|
||||||
from queue import Queue
|
|
||||||
import search_bot
|
|
||||||
import requests
|
|
||||||
from typing import Callable
|
|
||||||
from uuid import UUID
|
|
||||||
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
||||||
|
|
||||||
HOST_ADDRESS = "192.168.1.192"
|
HOST_ADDRESS = "192.168.1.192"
|
||||||
PORT_ADDRESS = 5001
|
PORT_ADDRESS = 5001
|
||||||
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
|
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
|
||||||
SEND_RESULTS = "/conjurer"
|
SEND_RESULTS = "/conjurer"
|
||||||
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
|
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
|
||||||
|
|
||||||
ENCODING = "utf-8"
|
ENCODING = "utf-8"
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
@@ -29,6 +31,7 @@ app = Flask(__name__)
|
|||||||
librarian_queue = Queue()
|
librarian_queue = Queue()
|
||||||
librarian_list = []
|
librarian_list = []
|
||||||
|
|
||||||
|
|
||||||
class Librarian(object):
|
class Librarian(object):
|
||||||
|
|
||||||
def __init__(self, app, query, uuid) -> None:
|
def __init__(self, app, query, uuid) -> None:
|
||||||
@@ -49,24 +52,24 @@ class Librarian(object):
|
|||||||
|
|
||||||
async def search_crossref(self, query):
|
async def search_crossref(self, query):
|
||||||
self.app.logger.info("STARTED SEARCH")
|
self.app.logger.info("STARTED SEARCH")
|
||||||
result = self.cr.works(query = query, limit = 1000)
|
result = self.cr.works(query=query, limit=1000)
|
||||||
self.search_result_from_cr.update(result)
|
self.search_result_from_cr.update(result)
|
||||||
self.total = result['message']['total-results']
|
self.total = result["message"]["total-results"]
|
||||||
self.fetched += len(result['message']['items'])
|
self.fetched += len(result["message"]["items"])
|
||||||
self.logger.info (self.total)
|
self.logger.info(self.total)
|
||||||
self.logger.info(self.fetched)
|
self.logger.info(self.fetched)
|
||||||
while self.total > self.fetched and self.limit > self.fetched:
|
while self.total > self.fetched and self.limit > self.fetched:
|
||||||
result = self.cr.works(query = query, limit = 1000)
|
result = self.cr.works(query=query, limit=1000)
|
||||||
self.search_result_from_cr.update(result)
|
self.search_result_from_cr.update(result)
|
||||||
self.total = result['message']['total-results']
|
self.total = result["message"]["total-results"]
|
||||||
self.fetched += len(result['message']['items'])
|
self.fetched += len(result["message"]["items"])
|
||||||
self.logger.info (self.total)
|
self.logger.info(self.total)
|
||||||
self.logger.info(self.fetched)
|
self.logger.info(self.fetched)
|
||||||
|
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
self.app.logger.info("CROSSREF DONE")
|
self.app.logger.info("CROSSREF DONE")
|
||||||
#TODO: add to sql with UUID
|
# TODO: add to sql with UUID
|
||||||
#sql row - title, doi, author, uuid, last retrieved, is_available
|
# sql row - title, doi, author, uuid, last retrieved, is_available
|
||||||
|
|
||||||
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
|
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
|
||||||
# First we load existing data into a dict.
|
# First we load existing data into a dict.
|
||||||
@@ -76,13 +79,29 @@ class Librarian(object):
|
|||||||
file_data = {}
|
file_data = {}
|
||||||
summarized_results = []
|
summarized_results = []
|
||||||
self.app.logger.info("REFINE: Removing all derived works from the list")
|
self.app.logger.info("REFINE: Removing all derived works from the list")
|
||||||
for item in self.search_result_from_cr['message']['items']:
|
for item in self.search_result_from_cr["message"]["items"]:
|
||||||
container = "NO"
|
container = "NO"
|
||||||
if "container-title" in item:
|
if "container-title" in item:
|
||||||
container = "YES"
|
container = "YES"
|
||||||
else:
|
else:
|
||||||
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
|
summarized_results.append(
|
||||||
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
|
{
|
||||||
|
"DOI": item["DOI"],
|
||||||
|
"title": item["title"] if "title" in item else None,
|
||||||
|
"type": item["type"] if "type" in item else None,
|
||||||
|
"container": container,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
self.uuid: {
|
||||||
|
"total_results": self.search_result_from_cr["message"][
|
||||||
|
"total-results"
|
||||||
|
],
|
||||||
|
"on_page": 1,
|
||||||
|
"summary": summarized_results,
|
||||||
|
"results": self.search_result_from_cr["message"]["items"],
|
||||||
|
}
|
||||||
|
}
|
||||||
if file_data:
|
if file_data:
|
||||||
file_data.update(result)
|
file_data.update(result)
|
||||||
else:
|
else:
|
||||||
@@ -96,7 +115,9 @@ class Librarian(object):
|
|||||||
|
|
||||||
async def check_dois_in_local_db(self, doi):
|
async def check_dois_in_local_db(self, doi):
|
||||||
self.app.logger.info("REFINE: Running search in the backend app")
|
self.app.logger.info("REFINE: Running search in the backend app")
|
||||||
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
|
coro = asyncio.to_thread(
|
||||||
|
search_bot.search_for_doi, doi, self.live_results, self.app.logger
|
||||||
|
)
|
||||||
result = await coro
|
result = await coro
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@@ -109,25 +130,34 @@ class Librarian(object):
|
|||||||
data = response.read()
|
data = response.read()
|
||||||
text = data.decode("utf-8")
|
text = data.decode("utf-8")
|
||||||
for line in text.splitlines():
|
for line in text.splitlines():
|
||||||
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
|
if re.match(
|
||||||
|
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
|
||||||
|
line,
|
||||||
|
):
|
||||||
return False
|
return False
|
||||||
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
|
if m := re.match(
|
||||||
|
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
|
||||||
|
):
|
||||||
direct_download_link = "https:" + str(m.group(1))
|
direct_download_link = "https:" + str(m.group(1))
|
||||||
self.app.logger.info (direct_download_link)
|
self.app.logger.info(direct_download_link)
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
async def check_if_exists(self, doi, brute_force = False):
|
async def check_if_exists(self, doi, brute_force=False):
|
||||||
result = {}
|
result = {}
|
||||||
result = await self.check_dois_in_local_db(doi)
|
result = await self.check_dois_in_local_db(doi)
|
||||||
if brute_force and not result:
|
if brute_force and not result:
|
||||||
for item in doi:
|
for item in doi:
|
||||||
item_link = "https://sci-hub.se/" + item[0]
|
item_link = "https://sci-hub.se/" + item[0]
|
||||||
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
|
tmp = {
|
||||||
|
"DOI": item["DOI"],
|
||||||
|
"exists": await self.check_if_exists_brute_force(item_link),
|
||||||
|
"data": item,
|
||||||
|
}
|
||||||
result.update(tmp)
|
result.update(tmp)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def refine_query(self, unrefined_reqult, brute_force = False):
|
async def refine_query(self, unrefined_reqult, brute_force=False):
|
||||||
temp = []
|
temp = []
|
||||||
refined_result = []
|
refined_result = []
|
||||||
for key in unrefined_reqult:
|
for key in unrefined_reqult:
|
||||||
@@ -149,20 +179,21 @@ class Librarian(object):
|
|||||||
refined_result = temp
|
refined_result = temp
|
||||||
self.hit = len(refined_result)
|
self.hit = len(refined_result)
|
||||||
return refined_result
|
return refined_result
|
||||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
|
|
||||||
|
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
|
||||||
async def answer_query(self):
|
async def answer_query(self):
|
||||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
|
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
|
||||||
#Search sql for UUID
|
# Search sql for UUID
|
||||||
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
|
# SQL SELECT * FROM RESULTS WHERE UUID = uuid
|
||||||
self.logger.info(f"Search started {self.uuid}")
|
self.logger.info(f"Search started {self.uuid}")
|
||||||
uuid_found = False
|
uuid_found = False
|
||||||
database = None
|
database = None
|
||||||
query = self.query
|
query = self.query
|
||||||
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
|
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
|
||||||
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
|
# tu logika komunikacji z SQL i cała logika związana z wyszukaniem
|
||||||
#ale tymczasowo plik
|
# ale tymczasowo plik
|
||||||
try:
|
try:
|
||||||
database = json.load(cr_file)
|
database = json.load(cr_file)
|
||||||
except JSONDecodeError:
|
except JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -175,7 +206,7 @@ class Librarian(object):
|
|||||||
database.update(answer)
|
database.update(answer)
|
||||||
cr_file.truncate(0)
|
cr_file.truncate(0)
|
||||||
cr_file.seek(0)
|
cr_file.seek(0)
|
||||||
json.dump(database,cr_file)
|
json.dump(database, cr_file)
|
||||||
self.result = await self.refine_query(answer)
|
self.result = await self.refine_query(answer)
|
||||||
return self.result
|
return self.result
|
||||||
else:
|
else:
|
||||||
@@ -183,13 +214,14 @@ class Librarian(object):
|
|||||||
database = answer
|
database = answer
|
||||||
cr_file.truncate(0)
|
cr_file.truncate(0)
|
||||||
cr_file.seek(0)
|
cr_file.seek(0)
|
||||||
json.dump(database,cr_file)
|
json.dump(database, cr_file)
|
||||||
self.result = await self.refine_query(answer)
|
self.result = await self.refine_query(answer)
|
||||||
self.app.logger.info("Refined")
|
self.app.logger.info("Refined")
|
||||||
self.done = True
|
self.done = True
|
||||||
return self.result
|
return self.result
|
||||||
|
|
||||||
#============================= FLASK INTERNALS===============================
|
|
||||||
|
# ============================= FLASK INTERNALS===============================
|
||||||
def flask_debug():
|
def flask_debug():
|
||||||
"""
|
"""
|
||||||
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
|
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
|
||||||
@@ -198,6 +230,7 @@ def flask_debug():
|
|||||||
# trunk-ignore(bandit/B201)
|
# trunk-ignore(bandit/B201)
|
||||||
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
def waitress_run():
|
def waitress_run():
|
||||||
"""
|
"""
|
||||||
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
||||||
@@ -205,6 +238,7 @@ def waitress_run():
|
|||||||
"""
|
"""
|
||||||
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
class BackgroundTaskSearch(threading.Thread):
|
class BackgroundTaskSearch(threading.Thread):
|
||||||
def run(self):
|
def run(self):
|
||||||
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
|
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
|
||||||
@@ -221,7 +255,7 @@ class BackgroundTaskSearch(threading.Thread):
|
|||||||
self.app.logger.info("Saving to file")
|
self.app.logger.info("Saving to file")
|
||||||
with open("s_results.json", "r+", encoding="utf-8") as s_file:
|
with open("s_results.json", "r+", encoding="utf-8") as s_file:
|
||||||
try:
|
try:
|
||||||
database = json.load(s_file)
|
database = json.load(s_file)
|
||||||
except JSONDecodeError:
|
except JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
if database:
|
if database:
|
||||||
@@ -236,17 +270,18 @@ class BackgroundTaskSearch(threading.Thread):
|
|||||||
|
|
||||||
self.app.logger.info(result)
|
self.app.logger.info(result)
|
||||||
coroutine = asyncio.to_thread(
|
coroutine = asyncio.to_thread(
|
||||||
requests.post,
|
requests.post,
|
||||||
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
|
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
|
||||||
json=result,
|
json=result,
|
||||||
timeout=360,
|
timeout=360,
|
||||||
)
|
)
|
||||||
self.app.logger.info("SENT")
|
self.app.logger.info("SENT")
|
||||||
_ = await coroutine
|
_ = await coroutine
|
||||||
self.app.logger.info("SEND CONFIRMED")
|
self.app.logger.info("SEND CONFIRMED")
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
#==================================SERVER ROUTES==========================================
|
|
||||||
|
# ==================================SERVER ROUTES==========================================
|
||||||
@app.route("/query", methods=["POST"])
|
@app.route("/query", methods=["POST"])
|
||||||
async def query_database():
|
async def query_database():
|
||||||
record = json.loads(request.data)
|
record = json.loads(request.data)
|
||||||
@@ -254,8 +289,8 @@ async def query_database():
|
|||||||
app.logger.info(record["query"])
|
app.logger.info(record["query"])
|
||||||
app.logger.info(record["UUID"])
|
app.logger.info(record["UUID"])
|
||||||
uuid = record["UUID"]
|
uuid = record["UUID"]
|
||||||
#TODO: TESTING ONLY======================
|
# TODO: TESTING ONLY======================
|
||||||
uuid = UUID('{12345678-1234-5678-1234-567812345678}')
|
uuid = UUID("{12345678-1234-5678-1234-567812345678}")
|
||||||
|
|
||||||
cl = Librarian(app, record["query"], uuid)
|
cl = Librarian(app, record["query"], uuid)
|
||||||
librarian_queue.put(cl)
|
librarian_queue.put(cl)
|
||||||
@@ -267,6 +302,7 @@ async def query_database():
|
|||||||
)
|
)
|
||||||
return return_data
|
return return_data
|
||||||
|
|
||||||
|
|
||||||
@app.route("/get_partial_result", methods=["POST"])
|
@app.route("/get_partial_result", methods=["POST"])
|
||||||
async def get_partial():
|
async def get_partial():
|
||||||
record = json.loads(request.data)
|
record = json.loads(request.data)
|
||||||
@@ -282,12 +318,13 @@ async def get_partial():
|
|||||||
)
|
)
|
||||||
return return_data
|
return return_data
|
||||||
|
|
||||||
#=======================================MAIN===================================================
|
|
||||||
|
# =======================================MAIN===================================================
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logger = logging.getLogger('conjurer_musician')
|
logger = logging.getLogger("conjurer_musician")
|
||||||
logger.setLevel(logging.DEBUG)
|
logger.setLevel(logging.DEBUG)
|
||||||
threads = []
|
threads = []
|
||||||
#threads.append(threading.Thread(target=waitress_run))
|
# threads.append(threading.Thread(target=waitress_run))
|
||||||
threads.append(threading.Thread(target=flask_debug))
|
threads.append(threading.Thread(target=flask_debug))
|
||||||
bgtask = BackgroundTaskSearch()
|
bgtask = BackgroundTaskSearch()
|
||||||
bgtask.app = app
|
bgtask.app = app
|
||||||
@@ -296,7 +333,7 @@ if __name__ == "__main__":
|
|||||||
for worker in threads:
|
for worker in threads:
|
||||||
try:
|
try:
|
||||||
logger.info(f"App number: {i}")
|
logger.info(f"App number: {i}")
|
||||||
i+=1
|
i += 1
|
||||||
worker.start()
|
worker.start()
|
||||||
except RuntimeError as e:
|
except RuntimeError as e:
|
||||||
logger.error("Exploded")
|
logger.error("Exploded")
|
||||||
|
|||||||
@@ -1,20 +1,22 @@
|
|||||||
|
# This Python file uses the following encoding: utf-8
|
||||||
"""
|
"""
|
||||||
The provided Python script sets up a Flask web server to manage a list of music files, with
|
The provided Python script sets up a Flask web server to manage a list of music files, with
|
||||||
functions for rescanning the music folder, updating the music list, and serving the music list via
|
functions for rescanning the music folder, updating the music list, and serving the music list via
|
||||||
API endpoints.
|
API endpoints.
|
||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
from pathlib import Path
|
|
||||||
from platform import uname
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from logging import handlers
|
||||||
|
from pathlib import Path
|
||||||
|
from platform import uname
|
||||||
from sys import platform
|
from sys import platform
|
||||||
import logging
|
|
||||||
from flask import Flask, jsonify, request
|
|
||||||
from waitress import serve
|
|
||||||
|
|
||||||
|
from flask import Flask, jsonify, redirect, request, send_from_directory
|
||||||
|
from waitress import serve
|
||||||
|
|
||||||
HOST_ADDRESS = "192.168.1.15"
|
HOST_ADDRESS = "192.168.1.15"
|
||||||
PORT_ADDRESS = 5000
|
PORT_ADDRESS = 5000
|
||||||
@@ -26,24 +28,42 @@ if platform in ("linux", "linux2"):
|
|||||||
NETRC_FILE = "/home/mtuszowski/.netrc"
|
NETRC_FILE = "/home/mtuszowski/.netrc"
|
||||||
LOGSTORE = "/home/mtuszowski/conjurer/logs/"
|
LOGSTORE = "/home/mtuszowski/conjurer/logs/"
|
||||||
ENCODING = "utf-8"
|
ENCODING = "utf-8"
|
||||||
GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
|
LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
|
||||||
MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
|
|
||||||
SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
|
|
||||||
MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
|
|
||||||
MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
|
|
||||||
SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
|
|
||||||
NETRC_FILE = "/home/pi/.netrc"
|
NETRC_FILE = "/home/pi/.netrc"
|
||||||
LOGSTORE = "/home/pi/RetroPie/logs/"
|
LOGSTORE = "/home/pi/RetroPie/logs/"
|
||||||
ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
|
|
||||||
ENCODING = "utf-8"
|
ENCODING = "utf-8"
|
||||||
GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
|
MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
|
||||||
DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
|
|
||||||
|
|
||||||
|
|
||||||
music_file_list = []
|
music_file_list = []
|
||||||
|
|
||||||
|
|
||||||
|
def create_playlist():
|
||||||
|
"""
|
||||||
|
Reads a JSON file containing a playlist and writes the playlist items to a text file.
|
||||||
|
|
||||||
|
The JSON file path is '/home/pi/Conjurer/playlist.json',
|
||||||
|
and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
JSONDecodeError: If the JSON file cannot be decoded.
|
||||||
|
|
||||||
|
"""
|
||||||
|
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
|
||||||
|
with open(
|
||||||
|
"/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
|
||||||
|
) as w_file:
|
||||||
|
try:
|
||||||
|
playlist = json.load(r_file)
|
||||||
|
for item in playlist:
|
||||||
|
w_file.write(item)
|
||||||
|
w_file.write("\n")
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def rescan():
|
def rescan():
|
||||||
"""
|
"""
|
||||||
The `rescan` function logs a message, scans for mp3 files in a specified folder,
|
The `rescan` function logs a message, scans for mp3 files in a specified folder,
|
||||||
@@ -55,6 +75,14 @@ def rescan():
|
|||||||
if platform == "win32":
|
if platform == "win32":
|
||||||
temp_music_file = temp_music_file.replace("/", "\\")
|
temp_music_file = temp_music_file.replace("/", "\\")
|
||||||
music_file_list.append(temp_music_file)
|
music_file_list.append(temp_music_file)
|
||||||
|
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
|
||||||
|
database = music_file_list
|
||||||
|
# self.app.logger.info("DUMPING DATA")
|
||||||
|
s_file.truncate(0)
|
||||||
|
s_file.seek(0)
|
||||||
|
json.dump(database, s_file)
|
||||||
|
create_playlist()
|
||||||
|
|
||||||
|
|
||||||
def thread_rescan():
|
def thread_rescan():
|
||||||
"""
|
"""
|
||||||
@@ -63,14 +91,31 @@ def thread_rescan():
|
|||||||
"""
|
"""
|
||||||
logging.info("Starting filesystemupdater")
|
logging.info("Starting filesystemupdater")
|
||||||
while True:
|
while True:
|
||||||
time.sleep(60*60*24)
|
time.sleep(60 * 60 * 24)
|
||||||
logging.info("Rescan triggered")
|
logging.info("Rescan triggered")
|
||||||
rescan()
|
rescan()
|
||||||
|
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
|
||||||
def wyszukaj(word_list, how_many):
|
|
||||||
logger = logging.getLogger()
|
# TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
|
||||||
|
def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
|
||||||
|
"""
|
||||||
|
Searches for music files based on a list of keywords and returns a list of matching files.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
word_list (list): A list of keywords to search for.
|
||||||
|
how_many (int): The maximum number of matching files to return.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list: A list of matching files, sorted by their search weight.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if _logger is None:
|
||||||
|
fun_logger = logging.getLogger()
|
||||||
|
else:
|
||||||
|
fun_logger = _logger
|
||||||
return_list = []
|
return_list = []
|
||||||
search_weight = []
|
search_weight = []
|
||||||
|
|
||||||
@@ -84,7 +129,7 @@ def wyszukaj(word_list, how_many):
|
|||||||
skip_start = 1
|
skip_start = 1
|
||||||
for word in word_list[skip_start:]:
|
for word in word_list[skip_start:]:
|
||||||
token_weight = len(word)
|
token_weight = len(word)
|
||||||
logger.info("Słowo kluczowe: %s", word)
|
fun_logger.info("Słowo kluczowe: %s", word)
|
||||||
itr = 0
|
itr = 0
|
||||||
for file in music_file_list:
|
for file in music_file_list:
|
||||||
file = file.split(SEPARATOR_FILE_PATH)
|
file = file.split(SEPARATOR_FILE_PATH)
|
||||||
@@ -124,6 +169,7 @@ def wyszukaj(word_list, how_many):
|
|||||||
else:
|
else:
|
||||||
skip = 4
|
skip = 4
|
||||||
matched_times = 1
|
matched_times = 1
|
||||||
|
# TODO: To do wątków albo for asynchroniczny
|
||||||
for itm in all_words[skip:]:
|
for itm in all_words[skip:]:
|
||||||
pingu += 1
|
pingu += 1
|
||||||
pattern = ".*" + word + ".*"
|
pattern = ".*" + word + ".*"
|
||||||
@@ -139,26 +185,42 @@ def wyszukaj(word_list, how_many):
|
|||||||
matched_times += 1
|
matched_times += 1
|
||||||
itr += 1
|
itr += 1
|
||||||
|
|
||||||
logger.info("Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start)
|
fun_logger.info(
|
||||||
|
"Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start
|
||||||
|
)
|
||||||
time_start = datetime.now()
|
time_start = datetime.now()
|
||||||
item_to_search = max_weight(search_weight)
|
item_to_search = max_weight(search_weight)
|
||||||
itr = 0
|
itr = 0
|
||||||
if item_to_search == 0:
|
if item_to_search == 0:
|
||||||
return None
|
return []
|
||||||
not_found = True
|
not_found = True
|
||||||
return_list = []
|
return_list = []
|
||||||
if int(how_many) <= 0:
|
if int(how_many) <= 0:
|
||||||
logger.info("Jeden plik do zagrania")
|
fun_logger.info("Jeden plik do zagrania")
|
||||||
while not_found:
|
while not_found:
|
||||||
if search_weight[itr][0] == item_to_search:
|
if search_weight[itr][0] == item_to_search:
|
||||||
return_list.append(search_weight[itr])
|
return_list.append(search_weight[itr])
|
||||||
return return_list
|
if not return_to_bot:
|
||||||
|
with open(
|
||||||
|
"/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
|
||||||
|
) as s_file:
|
||||||
|
s_file.write(search_weight[itr][1])
|
||||||
|
break
|
||||||
itr += 1
|
itr += 1
|
||||||
else:
|
else:
|
||||||
logger.info("Wiele plików do zagrania")
|
fun_logger.info("Wiele plików do zagrania")
|
||||||
search_weight.sort(key=lambda x: x[0], reverse=True)
|
search_weight.sort(key=lambda x: x[0], reverse=True)
|
||||||
return_list.extend(search_weight[: int(how_many)])
|
return_list.extend(search_weight[: int(how_many)])
|
||||||
return return_list
|
if not return_to_bot:
|
||||||
|
|
||||||
|
with open(
|
||||||
|
"/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
|
||||||
|
) as s_file:
|
||||||
|
s_file.truncate(0)
|
||||||
|
for item in return_list:
|
||||||
|
s_file.write(item[1] + "\n")
|
||||||
|
return return_list
|
||||||
|
|
||||||
|
|
||||||
def max_weight(lista):
|
def max_weight(lista):
|
||||||
"""
|
"""
|
||||||
@@ -188,6 +250,47 @@ def remove_characters(string, character):
|
|||||||
return string.replace(character, "")
|
return string.replace(character, "")
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/stream", methods=["GET"])
|
||||||
|
def stream_music():
|
||||||
|
"""
|
||||||
|
The function `get_music_list` returns a JSON object containing a list of music files.
|
||||||
|
:return: A JSON response containing a key "music_file_list" with the value of the variable
|
||||||
|
`music_file_list`.
|
||||||
|
"""
|
||||||
|
return send_from_directory("/tmp/hls", "stream.m3u8")
|
||||||
|
|
||||||
|
@app.route('/<string:file_name>')
|
||||||
|
def stream(file_name):
|
||||||
|
video_dir = '/tmp/hls'
|
||||||
|
return send_from_directory(video_dir, file_name)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/stream_mp3", methods=["GET"])
|
||||||
|
def stream_music_mp3():
|
||||||
|
"""
|
||||||
|
The function `get_music_list` returns a JSON object containing a list of music files.
|
||||||
|
:return: A JSON response containing a key "music_file_list" with the value of the variable
|
||||||
|
`music_file_list`.
|
||||||
|
"""
|
||||||
|
return redirect("http://www.example.com", code=302)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/clear_pr_pls", methods=["GET"])
|
||||||
|
def clear_pr_pls():
|
||||||
|
"""
|
||||||
|
The function `get_music_list` returns a JSON object containing a list of music files.
|
||||||
|
:return: A JSON response containing a key "music_file_list" with the value of the variable
|
||||||
|
`music_file_list`.
|
||||||
|
"""
|
||||||
|
with open(
|
||||||
|
"/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
|
||||||
|
) as s_file:
|
||||||
|
s_file.truncate(0)
|
||||||
|
return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
|
||||||
|
return return_data, 200
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@app.route("/mp3", methods=["GET"])
|
@app.route("/mp3", methods=["GET"])
|
||||||
def get_music_list():
|
def get_music_list():
|
||||||
"""
|
"""
|
||||||
@@ -218,6 +321,7 @@ def update_music_list():
|
|||||||
)
|
)
|
||||||
return return_data
|
return return_data
|
||||||
|
|
||||||
|
|
||||||
@app.route("/get_music", methods=["POST"])
|
@app.route("/get_music", methods=["POST"])
|
||||||
def look_for_playlist():
|
def look_for_playlist():
|
||||||
"""
|
"""
|
||||||
@@ -234,7 +338,31 @@ def look_for_playlist():
|
|||||||
app.logger.info(record["lista_slow"])
|
app.logger.info(record["lista_slow"])
|
||||||
app.logger.info(record["UUID"])
|
app.logger.info(record["UUID"])
|
||||||
app.logger.info(record["dlugosc_plejlisty"])
|
app.logger.info(record["dlugosc_plejlisty"])
|
||||||
return_data = wyszukaj(record["lista_slow"],record["dlugosc_plejlisty"])
|
return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True)
|
||||||
|
|
||||||
|
return_data = (
|
||||||
|
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
return return_data
|
||||||
|
|
||||||
|
@app.route("/add_to_priority", methods=["POST"])
|
||||||
|
def add_to_priority():
|
||||||
|
"""
|
||||||
|
The function `update_music_list` receives a POST request with a JSON payload, logs the received
|
||||||
|
item, adds it to a music file list, and returns a success message along with the updated record.
|
||||||
|
:return: The function `update_music_list` is returning a tuple containing a JSON response and a
|
||||||
|
status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
|
||||||
|
with values indicating the success of the operation and the data that was
|
||||||
|
received and added to the `music_file_list`.
|
||||||
|
The status code returned is 200, indicating a successful response.
|
||||||
|
"""
|
||||||
|
record = json.loads(request.data)
|
||||||
|
app.logger.info(record)
|
||||||
|
app.logger.info(record["lista_slow"])
|
||||||
|
app.logger.info(record["UUID"])
|
||||||
|
app.logger.info(record["dlugosc_plejlisty"])
|
||||||
|
return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False)
|
||||||
|
|
||||||
return_data = (
|
return_data = (
|
||||||
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
|
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
|
||||||
@@ -251,6 +379,7 @@ def flask_debug():
|
|||||||
# trunk-ignore(bandit/B201)
|
# trunk-ignore(bandit/B201)
|
||||||
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
def waitress_run():
|
def waitress_run():
|
||||||
"""
|
"""
|
||||||
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
||||||
@@ -258,12 +387,27 @@ def waitress_run():
|
|||||||
"""
|
"""
|
||||||
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logger = logging.getLogger('conjurer_musician')
|
logger = logging.getLogger("conjurer_musician")
|
||||||
logger.setLevel(logging.DEBUG)
|
logger.setLevel(logging.DEBUG)
|
||||||
|
formatter = logging.Formatter(
|
||||||
|
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||||
|
)
|
||||||
|
handler = handlers.RotatingFileHandler(
|
||||||
|
filename=LOGFILE,
|
||||||
|
encoding=ENCODING,
|
||||||
|
mode="a",
|
||||||
|
maxBytes=6 * 1024 * 1024,
|
||||||
|
backupCount=6,
|
||||||
|
)
|
||||||
|
handler.setFormatter(formatter)
|
||||||
|
logger.addHandler(handler)
|
||||||
|
|
||||||
rescan()
|
rescan()
|
||||||
logger.info("Started")
|
logger.info("Started")
|
||||||
threads = []
|
threads = []
|
||||||
|
#threads.append(threading.Thread(target=flask_debug))
|
||||||
threads.append(threading.Thread(target=waitress_run))
|
threads.append(threading.Thread(target=waitress_run))
|
||||||
threads.append(threading.Thread(target=thread_rescan))
|
threads.append(threading.Thread(target=thread_rescan))
|
||||||
|
|
||||||
@@ -272,5 +416,3 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
for worker in threads:
|
for worker in threads:
|
||||||
worker.join()
|
worker.join()
|
||||||
|
|
||||||
#TODO: Inne pliki - sadox, dokumenty naukowe itp.
|
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Conjurer_musician
|
||||||
|
After=multi-user.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/conjurer_musician.py
|
||||||
|
Restart=on-abort
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
Executable
+34804
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
|||||||
|
let json.parse credentials = file.contents("/home/pi/Conjurer/icecast_credentials.json")
|
||||||
|
enable_replaygain_metadata()
|
||||||
|
|
||||||
|
s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist"))
|
||||||
|
s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist"))
|
||||||
|
|
||||||
|
s = random(id="randomizer", weights=[7000, 100], [s2, s1])
|
||||||
|
|
||||||
|
interactive.harbor(port = 9999)
|
||||||
|
f = interactive.float("f", description="Frequency", min=0., max=1000.,unit="Hz", 200.)
|
||||||
|
g = interactive.float("g", description="Gain", min=0., max=20.,unit="dB", 8.)
|
||||||
|
b = bass_boost(frequency=f, gain=g, s)
|
||||||
|
s = add([s, b])
|
||||||
|
a = interactive.float("main_volume", min=0., max=20., 1.)
|
||||||
|
s = compress.multiband.interactive(bands=3, s)
|
||||||
|
|
||||||
|
s = nrj(normalize (s))
|
||||||
|
s = crossfade(fade_out=3., fade_in=3., duration=5., s)
|
||||||
|
s = amplify(a,s)
|
||||||
|
|
||||||
|
s = blank.skip(max_blank=2., s)
|
||||||
|
|
||||||
|
emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
|
||||||
|
radio = fallback(id="switcher",track_sensitive = false, [s, emergency])
|
||||||
|
|
||||||
|
handle_metadata = fun (m) -> begin
|
||||||
|
print(m["title"])
|
||||||
|
print(m["artist"])
|
||||||
|
print(m["filename"])
|
||||||
|
end
|
||||||
|
radio.on_metadata(handle_metadata)
|
||||||
|
interactive.persistent("script.params")
|
||||||
|
|
||||||
|
log_to_stdout = true
|
||||||
|
log_to_file = true
|
||||||
|
logpath = "/home/pi/Conjurer/radio_log.log"
|
||||||
|
loglevel = 3
|
||||||
|
|
||||||
|
# Enable logging on Standard Output and set logging level.
|
||||||
|
set("log.stdout",log_to_stdout)
|
||||||
|
set("log.level",loglevel)
|
||||||
|
|
||||||
|
# Enable logging to file. See variable: logpath
|
||||||
|
set("log.file",log_to_file)
|
||||||
|
set("log.file.path",logpath)
|
||||||
|
|
||||||
|
output.file.hls("/tmp/hls",[("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))],radio)
|
||||||
|
output.icecast(%mp3, host="retropie", port=8000,password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=radio_conjurer Liquidsoap daemon
|
||||||
|
After=network.target
|
||||||
|
Documentation=http://liquidsoap.fm/
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=pi
|
||||||
|
WorkingDirectory=/home/pi/Conjurer/
|
||||||
|
ExecStart=/home/pi/.opam/4.13.0/bin/liquidsoap /home/pi/Conjurer/radio_conjurer.liq
|
||||||
|
Restart=always
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
Alias=radio_conjurer-liquidsoap.service
|
||||||
Executable
+6
@@ -0,0 +1,6 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
cd ./conjurer
|
||||||
|
git pull
|
||||||
|
cd ..
|
||||||
|
cp ./conjurer/conjurer_musician/conjurer_musician.py ./Conjurer
|
||||||
|
sudo systemctl restart conjurer_musician.service
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
cd ./conjurer
|
||||||
|
git pull
|
||||||
|
cd ..
|
||||||
|
cp ./conjurer/conjurer_musician/radio_conjurer.liq ./Conjurer
|
||||||
|
sudo systemctl restart radio_service.service
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Conjurer_musician
|
|
||||||
After=multi-user.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
|
|
||||||
Restart=on-abort
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
python3 -m venv /home/pi/Conjurer-service/conjurer/file_webservice
|
|
||||||
source ./bin/activate
|
|
||||||
./bin/python3 -m pip install --upgrade pip
|
|
||||||
./bin/python3 -m pip install -r requirements_musician.txt
|
|
||||||
deactivate
|
|
||||||
|
|
||||||
sudo cp ./conjurer_musician.service /etc/systemd/system/
|
|
||||||
sudo systemctl daemon-reload
|
|
||||||
sudo systemctl start conjurer_musician.service
|
|
||||||
sudo systemctl enable conjurer_musician.service
|
|
||||||
+6
-2
@@ -1,5 +1,9 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
cd /home/pi
|
||||||
|
mdkir Conjurer
|
||||||
|
cd Conjurer
|
||||||
python3 -m venv /home/pi/Conjurer/env
|
python3 -m venv /home/pi/Conjurer/env
|
||||||
|
cp /home/pi/conjurer/requirements_bot.txt /home/pi/Conjurer/
|
||||||
source ./env/bin/activate
|
source ./env/bin/activate
|
||||||
./env/bin/python3 -m pip install --upgrade pip
|
./env/bin/python3 -m pip install --upgrade pip
|
||||||
./env/bin/python3 -m pip install -r requirements_bot.txt
|
./env/bin/python3 -m pip install -r requirements_bot.txt
|
||||||
@@ -8,5 +12,5 @@ sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.
|
|||||||
deactivate
|
deactivate
|
||||||
sudo cp ./conjurer.service /etc/systemd/system/
|
sudo cp ./conjurer.service /etc/systemd/system/
|
||||||
sudo systemctl daemon-reload
|
sudo systemctl daemon-reload
|
||||||
sudo systemctl start conjurer_musician.service
|
sudo systemctl start conjurer.service
|
||||||
sudo systemctl enable conjurer_musician.service
|
sudo systemctl enable conjurer.service
|
||||||
|
|||||||
Executable
+26
@@ -0,0 +1,26 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
cd /home/pi
|
||||||
|
mkdir Conjurer
|
||||||
|
cd Conjurer
|
||||||
|
python3 -m venv /home/pi/Conjurer/env
|
||||||
|
source /home/pi/Conjurer/env/bin/activate
|
||||||
|
cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjurer/
|
||||||
|
cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
|
||||||
|
cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
|
||||||
|
touch /home/pi/Conjurer/discord_mus_service.log
|
||||||
|
touch /home/pi/Conjurer/all_playlist.txt
|
||||||
|
touch /home/pi/Conjurer/playlist.json
|
||||||
|
touch /home/pi/Conjurer/priority_queue.playlist
|
||||||
|
touch /home/pi/Conjurer/radio_log.log
|
||||||
|
./env/bin/python3 -m pip install --upgrade pip
|
||||||
|
./env/bin/python3 -m pip install -r requirements_musician.txt
|
||||||
|
deactivate
|
||||||
|
sudo cp /home/pi/conjurer/conjurer_musician/conjurer_musician.service /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl start conjurer_musician.service
|
||||||
|
sudo systemctl enable conjurer_musician.service
|
||||||
|
|
||||||
|
sudo cp /home/pi/conjurer/conjurer_musician/radio_service.service /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl start radio_service.service
|
||||||
|
sudo systemctl enable radio_service.service
|
||||||
Reference in New Issue
Block a user