gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tag: 1.5

Browse Source 
Intermediate commits (oldest → newest):
- Hit list queue for radio
- fix
- Documentation of conjurer_librarian
- Reformatting of conjurer_librarian
- Documentation of conjurer_musician
- Formatting of conjurer_musician
- Documentation of radio_conjurer.liq
- Merge branch 'main' of https://github.com/migatu/conjurer
- Removal from tracking
- Fixing librarian result (#6)
- Potential bugfix for some lazycoding in spotipy... Another one. fuck it
- Music tunneling (#7)
- Even more logging in spotify download
- test requests
- Fine tuning
- Optimization bug fix #1
- Fix crossfade
- Crossfadeing ctnd
- Fix
- Starting work on librarian headless mode
This commit is contained in:
Polish Hammer
2025-10-30 16:59:00 +01:00
parent 806cada1c1
commit a8797df45c
20 changed files with 1161 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+4
View File
@@ -197,3 +197,7 @@ node_modules/prettier/esm/parser-postcss.mjs
node_modules/prettier/esm/parser-typescript.mjs
node_modules/prettier/esm/parser-yaml.mjs
node_modules/prettier/esm/standalone.mjs
cr_results.json
not_in_db.json
rr_results.json
s_results.json
bot.py
+195 -103
View File
@@ -44,7 +44,7 @@ import yt_dlp
from spotify_dl import spotify
from spotify_dl import youtube as youtube_download
from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q
from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q, QueryControl
# *=========================== Classes definition
@@ -62,22 +62,6 @@ Music_Config = TypedDict(
)
class QueryControl:
"""
This class `QueryControl` is used to manage queries with information about the author, UUID,
content, logger, context, and replies.
"""
def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
self.author = query_author
self.uuid = query_uuid
self.content = query_content
self.logger = uplogger
self.stop = False
self.ctx = ctx
self.logger.info(
f"Created Query control for {self.author}, {self.uuid}: {self.content}"
)
self.replies = []
class MusicFileList(object):
"""
@@ -131,43 +115,6 @@ class MusicFileList(object):
post_data = {"item": str(item)}
requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
class DoSearchView(discord.ui.View):
def __init__(self, _uuid, result_list, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.uuid = _uuid
self.result_list = result_list
@discord.ui.button(
label="Nastepna strona", style=discord.ButtonStyle.primary, row=0
)
async def next_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
@discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0)
async def reset_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
@discord.ui.button(
label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0
)
async def perv_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
# *=========================================== Predefines
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -192,6 +139,7 @@ GET_MP3 = "/mp3"
SEND_MP3 = "/update_mp3"
GET_PLAYLIST = "/get_music"
ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
REQUEST_MUSIC = "/request_radio_file"
CLEAR_PRIO = "/clear_pr_pls"
LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
SEND_QUERY = "/query"
@@ -351,9 +299,6 @@ async def on_ready():
@client.event
# trunk-ignore(pylint/R0915)
# trunk-ignore(pylint/R0914)
# trunk-ignore(pylint/R0912)
async def on_message(message):
"""
Handle incoming messages in a Discord server, perform various
@@ -607,8 +552,6 @@ async def disconnect(ctx):
await voice_client.disconnect()
# trunk-ignore(pylint/R0915)
# trunk-ignore(pylint/R0912)
async def play(ctx, zamawial=None, arg=None):
"""
Play a music file, retrieve metadata about the file, and send a
@@ -728,31 +671,41 @@ async def check_music():
await asyncio.sleep(2)
@tasks.loop(seconds=3)
# trunk-ignore(pylint/R0915)
async def check_data_q():
"""
This function checks the data queue for any new entries and processes them accordingly.
"""
try:
data = IN_COMM_Q.get(block=False)
fresh_data = IN_COMM_Q.get(block=False)
entries = []
if data.stop:
searcher = data.author
ctx = data.ctx
query = data.content
if fresh_data.stop:
searcher = fresh_data.author
query = fresh_data.content
l_p = 1
for doi in data.entries:
for doi in fresh_data.entries:
logger.info(doi)
desc = data.entries[doi]
desc = fresh_data.entries[doi]
title = desc['Title'][0]
entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n")
l_p+=1
message = '*Z podłogi wysuwa się winda na książki*'
message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query}'
if fresh_data.ctx is not None:
ctx = fresh_data.ctx
message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}'
else:
ctx = client.get_channel(1062047571557744721)
message += f' Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał'
message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
if len(entries)<1:
message += "niestety nie ma nic"
await ctx.send(message)
elif len(entries)>=1 and len(entries)<5:
message += " znajduje się coś:\n"
for item in entries:
message+=item
await ctx.send(message)
else:
message+= " znajduje się cholernie dużo:\n"
for item in entries:
@@ -761,20 +714,17 @@ async def check_data_q():
await ctx.send(message)
message = ""
await ctx.send(message)
#Kept for sentimental reasons
#await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
except Empty:
pass
@tasks.loop(seconds=120)
# trunk-ignore(pylint/R0915)
async def check_self():
"""
The function `check_data` periodically checks for conditions to send messages and manage log files
in a Discord channel.
"""
# trunk-ignore(codespell/misspelled)
# logger.info("Heartbeat of cleanup proc")
channel = client.get_channel(1062047367337095268)
messages = [message async for message in channel.history(limit=1)]
@@ -837,9 +787,7 @@ async def check_self():
await channel.send(
"*Przeciąga się za barem* No dobra - porobione to można dalej pi... *Zauważa spojrzenie Hammera* eprzyć o głupotach z klientami... Szefie... *Bierze 'ukradkowy' łyk z piersiówki*"
)
# trunk-ignore(pylint/W0603)
global LAST_SPONTANEOUS_CALL
# trunk-ignore(pylint/W0603)
global TIME_BETWEEN_CALLS
tdelta = datetime.now() - LAST_SPONTANEOUS_CALL
tdelta = tdelta.total_seconds()
@@ -875,7 +823,6 @@ async def get_random_cyclic_message():
mess_key = list(cyclic_words.keys())[messnum]
return cyclic_words[mess_key][0]
ai_check2 = random.randint(0, 10)
# trunk-ignore(pylint/W0603)
global MESSAGE_TABLE
if ai_check2 < 6:
logger.info("Dykteryjka")
@@ -909,10 +856,6 @@ async def get_random_cyclic_message():
return result
# trunk-ignore(pylint/R0912)
# trunk-ignore(pylint/R0913)
# trunk-ignore(pylint/R0914)
# trunk-ignore(pylint/R0915)
async def handle_response(
prompt, vykidailo, bartender, history, username, request_type
):
@@ -1101,8 +1044,6 @@ async def handle_response(
return result, MESSAGE_TABLE
# trunk-ignore(pylint/R0915)
# trunk-ignore(pylint/R0914)
async def get_file(ctx, source, link):
"""
Take in three parameters: "ctx" (context),
@@ -1124,11 +1065,15 @@ async def get_file(ctx, source, link):
if source == "Spotify":
dir_path = None
item_type, item_id = spotify.parse_spotify_url(link)
if item_id:
coro = asyncio.to_thread(spotify.fetch_tracks, args=(spotify_ctrl, item_type, link))
file_list = await coro
coro = asyncio.to_thread(spotify.get_item_name, args=(spotify_ctrl, item_type, item_id))
directory_name = await coro
if item_type in ["album", "track", "playlist"] and item_id:
logger.info(item_type)
logger.info(item_id)
logger.info("Spotify link provided")
file_list = await asyncio.to_thread(spotify.fetch_tracks, spotify_ctrl, item_type, link)
logger.info(file_list)
directory_name = await asyncio.to_thread(spotify.get_item_name, spotify_ctrl, item_type, item_id)
logger.info(directory_name)
logger.info("Spotify scrape done")
url_data = {"urls": []}
url_dict = {}
url_dict["save_path"] = Path(
@@ -1138,6 +1083,7 @@ async def get_file(ctx, source, link):
url_dict["songs"] = file_list
url_data["urls"].append(url_dict.copy())
file_name_f = youtube_download.default_filename
logger.info("YT-DL")
coro = asyncio.to_thread(
youtube_download.download_songs,
songs=url_data,
@@ -1153,6 +1099,7 @@ async def get_file(ctx, source, link):
proxy="",
)
_ = await coro
logger.info("YT-DL done")
dir_path = (url_dict["save_path"].resolve()).as_posix()
if platform == "win32":
dir_path = dir_path.replace("/", "\\")
@@ -1323,9 +1270,6 @@ async def get_stats(ctx, history_limit):
yield name, how_many
# trunk-ignore(pylint/R0915)
# trunk-ignore(pylint/R0912)
# trunk-ignore(pylint/R0914)
async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None):
"""
Take in a context object and an optional integer parameter "how_many" and perform search
@@ -1485,8 +1429,6 @@ async def dej_co_z_jutuba(ctx):
description="Jeśli nie wiesz jak użyć tej komendy to nawet nie próbuj",
guild=discord.Object(id=664789470779932693),
)
# trunk-ignore(pylint/R0914)
# trunk-ignore(pylint/R0912)
async def update_banlist(ctx):
"""
Update a banlist in a Discord guild from preprovided list in channel.
@@ -1638,6 +1580,40 @@ async def graj_muzyko(ctx):
logger.info("Press play on tape completed")
check_music.start()
@client.hybrid_command(
name="radio_hardkor",
description="Włącza radio Conjurer na kanale #nocna-zmiana.",
guild=discord.Object(id=664789470779932693),
)
async def radio_hardkor(ctx):
"""
Plays the radio hardkor stream in the voice channel.
Args:
ctx: The context object representing the invocation context.
Returns:
None
"""
logger.info("Press play on radio hardkor")
async with ctx.typing():
global MUZYKA # pylint: disable=global-variable-not-assigned
MUZYKA["ctx"] = ctx
if not client.voice_clients:
await connect(ctx=ctx)
voice_client = client.voice_clients[0]
voice_client.play(
discord.PCMVolumeTransformer(
discord.FFmpegPCMAudio("http://95.175.16.246:666/mp3-stream"), volume=0.3
)
)
logger.info("Press play on radio completed")
else:
logger.error("Already playing")
await asyncio.sleep(12)
check_music.start()
@client.hybrid_command(
name="cisza",
@@ -1813,7 +1789,36 @@ async def dodaj_do_ulubionych(ctx):
timeout=360,
)
_ = await coroutine
@client.hybrid_command(
name="ja_chciol",
description="Dodaje do listy ulubionych w radiu",
guild=discord.Object(id=664789470779932693),
)
async def request_radio(ctx):
"""
Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
Rest of the line are search terms.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. This parameter is required in all
Discord.py commands
"""
async with ctx.typing():
logger.info("Zaczynam szukać timestamp %s", datetime.now())
word_list = ctx.message.content.split()
jrequest = {
"lista_slow": word_list,
"UUID": str(uuid.uuid4()),
}
coroutine = asyncio.to_thread(
requests.post,
f"{FILE_SERVICE_ADDRESS}{REQUEST_MUSIC}",
json=jrequest,
timeout=360,
)
_ = await coroutine
@client.hybrid_command(
name="wyczysc_ulubione",
description="Czysci liste ulubionych w radiu",
@@ -1831,9 +1836,8 @@ async def wyczysc_ulubione(ctx):
"""
async with ctx.typing():
coroutine = asyncio.to_thread(
requests.post,
requests.get,
f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
json={},
timeout=360,
)
_ = await coroutine
@@ -2058,8 +2062,6 @@ async def get_image_stable_diffusion(ctx):
description="Gra muzyke wyszukana na bazie tematow konwersacji na kanale NZ",
guild=discord.Object(id=664789470779932693),
)
# trunk-ignore(pylint/R0912)
# trunk-ignore(pylint/R0915)
async def zagraj_muzyke_mego_ludu(ctx):
"""
Play completeley random playlist based on messaging history.
@@ -2209,11 +2211,20 @@ async def parametry_muzyki_mego_ludu(
guild=discord.Object(id=664789470779932693),
)
async def wyszukaj_linki_do_dokumentow(ctx):
"""
The function `wyszukaj_linki_do_dokumentow` searches for links to documents in a crossref database and provides links to scihub.
:param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots.
It represents the context of the command being executed, including information about the message, the server,
and the user who invoked the command.
"""
query = ctx.message.content
query_uuid = uuid.uuid4()
#TODO: TESTING ONLY!!
#query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
json_query = {"UUID": str(query_uuid), "query": str(query)[30:], "page": 1}
ctx.message.content = ctx.message.content.replace("$wyszukaj_linki_do_dokumentow", "")
json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": False}
coroutine = asyncio.to_thread(
requests.post,
f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
@@ -2222,8 +2233,8 @@ async def wyszukaj_linki_do_dokumentow(ctx):
)
await ctx.send(
"*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
" wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to te 3/4 stacji."+
" A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc te 12/8 stacji teraz będzie ciężko pracować"
" wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."+
" A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować"
)
query_response = await coroutine
if not query_response.status_code == 200:
@@ -2245,6 +2256,87 @@ async def wyszukaj_linki_do_dokumentow(ctx):
await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+
" Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.")
@client.hybrid_command(
name="glebokie_gardlo",
description="Przygotowuje drinka o nazwie głębokie gardło",
guild=discord.Object(id=664789470779932693),
)
async def wyszukaj_linki_do_dokumentow_deep(ctx):
"""
The function `wyszukaj_linki_do_dokumentow_deep` performs a deep search for links to documents in
a crossref database and provides links to scihub.
:param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots.
It represents the context of the command being executed, including information about the message, the server,
and the user who invoked the command.
"""
# TODO: Implement deep search logic here
query = ctx.message.content.replace("$glebokie_gardlo", "")
allowed = False
for role in ctx.message.author.roles:
if role.name == "Bartender":
allowed = True
if role.name == "Scribe":
allowed = True
if role.name == "Thane":
allowed = True
if not allowed:
if ctx.message.author.nick:
username = ctx.message.author.nick
else:
username = ctx.message.author.name
vykidailo = False
bartender = False
prompt = "Przygotuj mi drinka o nazwie Głębokie Gardło, inspirowanego tym sławnym filmem oraz skandalem Watergate"
for role in ctx.message.author.roles:
if role.name == "Vykidailo":
vykidailo = True
if role.name == "Bartender":
bartender = True
global MESSAGE_TABLE # pylint: disable=global-statement
result, MESSAGE_TABLE = await handle_response(
prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION"
)
await ctx.send(result)
return
query_uuid = uuid.uuid4()
#TODO: TESTING ONLY!!
#query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": True}
coroutine = asyncio.to_thread(
requests.post,
f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
json=json_query,
timeout=360,
)
await ctx.send(
"*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
" wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"+
" Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego"
)
query_response = await coroutine
if not query_response.status_code == 200:
await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+
" Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało")
return
query, query_uuid, queue_size = (
query_response.json()["data"][0],
query_response.json()["data"][1],
query_response.json()["data"][2],
)
if ctx.message.author.nick:
username = ctx.message.author.nick
else:
username = ctx.message.author.name
query_object = QueryControl(username, query_uuid, query, logger, ctx)
OUT_COMM_Q.put(query_object)
await ctx.send(f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."+
" Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*")
@client.hybrid_command(
name="krecimy_pornola",
@@ -2289,16 +2381,16 @@ if __name__ == "__main__":
threads.append(threading.Thread(target=client.run, args=(TOKEN,)))
threads.append(threading.Thread(target=comm_subroutine, args=(logger,)))
logger.info("Starting discord bot: Starting threads")
i = 0
WRK_CNT = 0
for worker in threads:
i+=1
logger.info("Starting discord bot: Starting thread {i}")
WRK_CNT+=1
logger.info("Starting discord bot: Starting thread %s", WRK_CNT)
worker.start()
logger.info("Starting discord bot: Joining threads")
i=0
WRK_CNT=0
for worker in threads:
i+=1
logger.info("Starting discord bot: Joining thread {i}")
WRK_CNT+=1
logger.info("Starting discord bot: Joining thread %s", WRK_CNT)
worker.join()
# widok - generuje się 5 stron po 4 linki
cogs/music.py
+46
View File
@@ -0,0 +1,46 @@
import discord
from discord.ext import commands
from typing import Optional
class Music(commands.Cog):
def __init__(self, bot):
self.bot = bot # This is so you can access Bot instance in your cog
# You must have this function for `bot.load_extension` to call
def setup(bot):
bot.add_cog(Music(bot))
@commands.hybrid_command(
name="przytul", description="Przytul kogoś - daj mention po komendzie :)"
)
async def przytul(ctx, arg: Optional[discord.Member] = None):
"""
Generate a text about hugging mentioned user.
:param ctx: ctx stands for "context" and is a required parameter in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command
:param arg: arg is a parameter of the function "przytul" that expects a Discord member object. The
parameter is optional, meaning that if no member object is provided, it will default to None
:type arg: Optional[discord.Member]
"""
async with ctx.typing():
nieprzytulac = False
for mention in ctx.message.mentions:
for role in mention.roles:
if role.name == "NIEPRZYTULAĆ!":
nieprzytulac = True
if arg and nieprzytulac:
await ctx.send(
f"Żebym ja Ciebie nie przytulił {ctx.message.author.mention}"
)
elif arg:
await ctx.send(
# trunk-ignore(codespell/misspelled)
f"Już dobrze.... Już dobrze... Ojej.. Biedactwo... :( *W ułamku sekundy {arg.mention} znajduje sie w duszącym uścisku. Żebra trzeszczą - kilka pęka. Pacnięcia po plecach grożą odbiciem nerek, a głaskanie po głowie powoduje wstrząs mózgu*"
)
else:
await ctx.send(
"Kogo mam przytulić? *Wyłamuje kostki i przeciąga się - jego 200 kilowa sylwetka złożona z samych mięśni świadczy o tym że jest gotowy*"
)
communication_subroutine.py
+24
View File
@@ -13,6 +13,22 @@ IN_COMM_Q = Queue()
awaiting_q = []
incoming_q = Queue()
app = Flask(__name__)
class QueryControl:
"""
This class `QueryControl` is used to manage queries with information about the author, UUID,
content, logger, context, and replies.
"""
def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
self.author = query_author
self.uuid = query_uuid
self.content = query_content
self.logger = uplogger
self.stop = False
self.ctx = ctx
self.logger.info(
f"Created Query control for {self.author}, {self.uuid}: {self.content}"
)
self.replies = []
@app.route("/conjurer", methods=["POST"])
def answer_external_command():
@@ -86,8 +102,16 @@ def scan_incoming(_logger):
try:
answer = incoming_q.get(block=False)
_logger.info("DATA FOUND")
record_stored = False
for record in awaiting_q:
if record.uuid in answer.keys():
record_stored = True
record.stop = True
record.entries = answer[record.uuid]
IN_COMM_Q.put(record)
if not record_stored:
for key in answer.keys():
record = QueryControl("Orphaned", key, "Orphan", _logger, None)
record.stop = True
record.entries = answer[record.uuid]
IN_COMM_Q.put(record)
conjurer_librarian/conjurer_librarian.py
+345 -186
View File
@@ -1,243 +1,378 @@
from logging import handlers
import re
"""
This module contains the implementation of the Librarian class and
related functions for searching and refining queries.
Classes:
- Librarian: Represents a librarian object that performs search and
refinement operations on queries
Functions:
- flask_debug: Starts a Flask application in debug mode without using the reloader.
- waitress_run: Serves the Flask application using the Waitress WSGI server.
- BackgroundTaskSearch: Represents a background task for running the
Librarian object asynchronously
"""
import asyncio
import json
import logging
import netrc
import threading
import time
from flask import Flask, jsonify, request
import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from logging import handlers
from queue import Queue
import requests
import scrape_bot
import search_bot
from flask import Flask, jsonify, request
from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
import scrape_bot
import requests
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
# Constants
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
MAX_CR_RESULTS = 150000
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
MAX_CR_RESULTS = 7000
ENCODING = "utf-8"
app = Flask(__name__)
librarian_queue = Queue()
librarian_list = []
class Librarian(object):
def __init__(self, app, query, uuid) -> None:
# trunk-ignore(pylint/R0902)
class Librarian(object):
"""
Represents a librarian object that performs search and refinement operations on queries.
"""
def __init__(self, _app, query, uuid, _deep_search) -> None:
"""
Initializes a Librarian object.
Args:
- _app: The Flask application object.
- query: The query to be searched.
- uuid: The unique identifier for the search.
Attributes:
- cr: The Crossref object for performing the search.
- query: The query to be searched.
- uuid: The unique identifier for the search.
- limit: The maximum number of search results to fetch.
- fetched: The number of search results fetched so far.
- hit: The number of search results that match the refinement criteria.
- total: The total number of search results.
- app: The Flask application object.
- live_results: A list to store live search results.
- final_result: A list to store the final refined search results.
- not_in_db: A list to store search results that are not in the local database.
- search_result_from_cr: A dictionary to store the search results from Crossref.
- done: A flag indicating if the search is done.
"""
netrc_mod = netrc.netrc(NETRC_FILE)
authTokens = netrc_mod.authenticators("crossref")
self.cr = Crossref(mailto=authTokens[0])
auth_tokens = netrc_mod.authenticators("crossref")
self.cr = Crossref(
mailto=auth_tokens[0],
ua_string=f"Conjurer project. mailto:{auth_tokens[0]}"
)
self.query = query
self.uuid = str(uuid)
self.limit = MAX_CR_RESULTS
self.fetched = 0
self.hit = 0
self.total = 0
self.app = app
self.app = _app
self.live_results = []
self.final_result = []
self.not_in_db = []
self.final_result = {}
self.not_in_db = {}
self.search_result_from_cr = {}
self.done = False
self.deep_search = _deep_search
async def search_crossref(self, query, deep_search=False):
"""
Performs a search on Crossref for the given query.
Args:
- query: The query to be searched.
Returns:
- result: The search result from Crossref.
Raises:
- None.
"""
async def search_crossref(self, query):
self.app.logger.info("STARTED SEARCH")
result = self.cr.works(query = query, limit = 1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.app.logger.info (self.total)
self.app.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched:
result = self.cr.works(query = query, limit = 1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.app.logger.info (self.total)
self.app.logger.info(self.fetched)
time.sleep(0.1)
self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
if not deep_search:
cr_result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(cr_result)
self.total = cr_result["message"]["total-results"]
self.fetched += len(cr_result["message"]["items"])
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched:
tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched)
cr_result["message"]["items"].extend(tmp_result["message"]["items"])
self.total = tmp_result["message"]["total-results"]
self.fetched = len(cr_result["message"]["items"])
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
time.sleep(0.1)
else:
cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True)
result = cr_result[0]
for item in cr_result[1:]:
result["message"]["items"].extend(item["message"]["items"])
self.total = item["message"]["total-results"]
self.fetched = len(result["message"]["items"])
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
cr_result = result
self.app.logger.info("Total, fetched:")
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
self.search_result_from_cr.update(cr_result)
self.total = cr_result["message"]["total-results"]
self.app.logger.info("CROSSREF DONE")
self.app.logger.info("CROSSREF DONE")
with open("cr_results.json", "r+", encoding="utf-8") as data_file:
# First we load existing data into a dict.
try:
file_data = json.load(data_file)
except JSONDecodeError:
file_data = {}
data_file.truncate(0)
data_file.seek(0)
tmp = {self.uuid : self.search_result_from_cr}
if file_data:
file_data.update(tmp)
else:
file_data = tmp
json.dump(file_data, data_file, indent=4)
return cr_result
async def refine_search(self, unrefined_result):
"""
Refines the search query based on the unrefined search result.
Args:
- unrefined_result: The unrefined search result.
Returns:
- refined_result: The refined search result.
Raises:
- None.
"""
summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list")
for item in unrefined_result["message"]["items"]:
summarized_results.append(
{
"DOI": item["DOI"],
"title": item["title"] if "title" in item else None,
"type": item["type"] if "type" in item else None,
}
)
partial_result = {
self.uuid: {
"total_results": self.search_result_from_cr["message"][
"total-results"
],
"on_page": 1,
"summary": summarized_results,
"results": self.search_result_from_cr["message"]["items"],
}
}
self.app.logger.info("REFINE: Dumping to file")
temp = []
refined_result = {}
for key in partial_result:
self.app.logger.info("KEY:")
self.app.logger.info(key)
for item in partial_result[self.uuid]["summary"]:
if item["title"]:
temp.append(item)
for item in temp:
refined_result[item["DOI"]]= item
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
# First we load existing data into a dict.
try:
file_data = json.load(data_file)
except JSONDecodeError:
file_data = {}
summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list")
for item in self.search_result_from_cr['message']['items']:
container = "NO"
if "container-title" in item:
container = "YES"
else:
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
if file_data:
file_data.update(result)
else:
file_data = result
self.app.logger.info("REFINE: Dumping to file")
data_file.truncate(0)
data_file.seek(0)
tmp = {self.uuid: refined_result}
if file_data:
file_data.update(tmp)
else:
file_data = tmp
json.dump(file_data, data_file, indent=4)
return result
return refined_result
async def check_dois_in_local_db(self, doi):
self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
result = await coro
return result
async def check_if_exists(self, refined_result):
"""
Checks if the given DOI exists.
async def check_if_exists_brute_force(self, page_url):
self.app.logger.error("REFINE: Brute force search!")
if not page_url.startswith(("http:", "https:")):
raise ValueError("URL must start with 'http:' or 'https:'")
# trunk-ignore(bandit/B310)
with urlopen(page_url) as response:
data = response.read()
text = data.decode("utf-8")
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
return False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
direct_download_link = "https:" + str(m.group(1))
self.app.logger.info (direct_download_link)
return True
return False
Args:
- doi: The DOI to be checked.
- brute_force: A flag indicating if brute force method should be used.
async def check_if_exists(self, doi, brute_force = False):
Returns:
- result: The search result.
Raises:
- None.
"""
result = {}
result = await self.check_dois_in_local_db(doi)
if brute_force and not result:
for item in doi:
item_link = "https://sci-hub.se/" + item[0]
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
result.update(tmp)
return result
async def refine_query(self, unrefined_reqult, brute_force = False):
temp = []
refined_result = []
for key in unrefined_reqult:
self.app.logger.info("KEY:")
self.app.logger.info(key)
for item in unrefined_reqult[self.uuid]["summary"]:
if item["container"] == "NO" and item["title"]:
refined_result.append(item)
for item in refined_result:
temp.append([item["DOI"], item])
refined_result = await self.check_if_exists(temp)
for item in refined_result:
self.app.logger.info("REFINE: Running search in the backend app")
dois = []
for item, value in refined_result.items():
dois.append([item, value])
coro = asyncio.to_thread(
search_bot.search_for_doi, dois, self.live_results, self.app.logger
)
result = await coro
result_list = []
result_no_db = []
for item in result:
if item["exists"]:
self.final_result.append(item["data"])
result_list.append(item)
else:
self.not_in_db.append(item["data"])
self.hit = len(self.final_result)
return self.final_result
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
async def answer_query(self):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
self.app.logger.info(f"Search started {self.uuid}")
database = None
query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
#ale tymczasowo plik
try:
database = json.load(cr_file)
except JSONDecodeError:
pass
#TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
if database:
for item in database.keys():
if item in self.uuid:
self.result = await self.refine_query(database)
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
answer = await self.search_crossref(query=query)
database.update(answer)
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
self.result = await self.refine_query(answer)
else:
answer = await self.search_crossref(query=query)
database = answer
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
self.result = await self.refine_query(answer)
self.app.logger.info("Refined")
self.done = True
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
result_no_db.append(item)
self.hit = len(result)
return result_list, result_no_db
async def answer_query(self, deep_search=False):
"""
Answers the search query.
Args:
- None.
Returns:
- result: The search result.
Raises:
- None.
"""
self.app.logger.info(f"Search started {self.uuid}")
cr_result = await self.search_crossref(query=self.query, deep_search=deep_search)
refined_result = await self.refine_search(cr_result)
answer, negative_answer = await self.check_if_exists(refined_result)
self.app.logger.info("Returning result")
self.app.logger.info(answer)
self.app.logger.info(negative_answer)
for item in answer:
self.final_result[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
for item in negative_answer:
self.not_in_db[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
self.app.logger.info("Returning result case2")
self.app.logger.info(self.final_result)
return self.final_result
# ============================= FLASK INTERNALS===============================
#============================= FLASK INTERNALS===============================
def flask_debug():
"""
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
Do not use for production for fucks sake.
Starts a Flask application in debug mode without using the reloader.
Args:
- None.
Returns:
- None.
Raises:
- None.
"""
# trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run():
"""
The `waitress_run` function serves the `app` on host "0.0.0.0"
and port 5000 using the Waitress WSGI server.
Serves the Flask application using the Waitress WSGI server.
Args:
- None.
Returns:
- None.
Raises:
- None.
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
class BackgroundTaskSearch(threading.Thread):
"""
A background task for searching and saving results to files.
This class extends the `threading.Thread` class and is responsible for running
the search task in the background. It retrieves queries from a queue, performs
the search, and saves the results to files.
Attributes:
app (App): The application instance.
"""
def run(self):
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
"""
Run the background task.
This method is called when the thread is started. It creates a new event loop,
runs the `_run` method, and closes the event loop.
"""
loop = asyncio.new_event_loop()
loop.run_until_complete(self._run())
loop.close()
async def _run(self):
"""
Perform the search task.
This method is an asynchronous coroutine that runs in a loop. It retrieves a
librarian from the queue, answers the query, and saves the results to files.
It also sends the results to a remote server.
The search task continues running indefinitely until the thread is stopped.
"""
while True:
database = None
ndb_database = None
librarian = librarian_queue.get()
self.app.logger.info("STARTED")
result = await librarian.answer_query()
result = await librarian.answer_query(librarian.deep_search)
result = {librarian.uuid: result}
self.app.logger.info("Saving to file")
# Save results to "not_in_db.json" file
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
ndb_database = json.load(ndb_file)
except JSONDecodeError:
pass
if ndb_database:
@@ -248,9 +383,10 @@ class BackgroundTaskSearch(threading.Thread):
ndb_file.seek(0)
json.dump(ndb_database, ndb_file)
# Save results to "s_results.json" file
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
database = json.load(s_file)
except JSONDecodeError:
pass
if database:
@@ -267,26 +403,41 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info(result)
coroutine = asyncio.to_thread(
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
self.app.logger.info("SENT")
result = await coroutine
self.app.logger.info(result.status_code)
self.app.logger.info("SEND CONFIRMED")
time.sleep(1)
#==================================SERVER ROUTES==========================================
# ==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"])
async def query_database():
"""
Endpoint for querying the database.
This function receives a POST request containing a JSON payload with a query and a UUID.
It creates a Librarian object with the query and UUID,
and adds it to the librarian_queue and librarian_list.
Finally, it returns a JSON response indicating the success
of the operation, along with the query, UUID,
and the current size of the librarian_queue.
Returns:
tuple: A tuple containing a JSON response and a status code.
"""
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["query"])
app.logger.info(record["UUID"])
uuid = record["UUID"]
cl = Librarian(app, record["query"], uuid)
deep_search = record["deep_search"]
cl = Librarian(app, record["query"], uuid, deep_search)
librarian_queue.put(cl)
librarian_list.append(cl)
answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
@@ -296,8 +447,15 @@ async def query_database():
)
return return_data
@app.route("/get_partial_result", methods=["POST"])
async def get_partial():
"""
Retrieves the partial result for a given UUID.
Returns:
A JSON response containing the partial result.
"""
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["UUID"])
@@ -311,30 +469,31 @@ async def get_partial():
)
return return_data
#=======================================MAIN===================================================
# =======================================MAIN===================================================
if __name__ == "__main__":
app.logger.setLevel(logging.DEBUG)
h1 = handlers.RotatingFileHandler(
filename="E:\\logs\\librarian.log",
encoding="utf-8",
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
filename="D:\\logs\\librarian.log",
encoding="utf-8",
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
)
app.logger.addHandler(h1)
threads = []
threads.append(threading.Thread(target=waitress_run))
#threads.append(threading.Thread(target=flask_debug))
# threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch()
bgtask.app = app
threads.append(bgtask)
threads.append(threading.Thread(target = scrape_bot.scraper, args=(app.logger,)))
threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
i = 0
for worker in threads:
try:
app.logger.info(f"App number: {i}")
i+=1
app.logger.info("App number: %s", i)
i += 1
worker.start()
except RuntimeError as e:
app.logger.error("Exploded")
conjurer_librarian/install_librarian_service.sh
+31 -9
View File
@@ -1,11 +1,33 @@
#!/bin/bash
python3 -m venv /home/pi/Conjurer_librarian/env
cp /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
cp /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
cd /home/pi/Conjurer_librarian
source /home/pi/Conjurer_librarian/env/bin/activate
./env/bin/python3 -m pip install --upgrade pip
./env/bin/python3 -m pip install -r requirements_librarian.txt
deactivate
# This script sets up a virtual environment for the Conjurer Librarian service and installs the required dependencies.
cp /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
# Create a virtual environment at /home/pi/Conjurer_librarian/env
python3 -m venv /home/pi/Conjurer_librarian/env
# Copy the Conjurer Librarian Python script to the appropriate directory
cp /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
# Copy the requirements file to the appropriate directory
cp /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
# Change directory to /home/pi/Conjurer_librarian
cd /home/pi/Conjurer_librarian
touch /home/pi/Conjurer/rr_results.json
touch /home/pi/Conjurer/cr_results.json
touch /home/pi/Conjurer/s_results.json
touch /home/pi/Conjurer/rr_results.json
touch /home/pi/Conjurer/not_in_db.json
# Activate the virtual environment
source /home/pi/Conjurer_librarian/env/bin/activate
# Upgrade pip
./env/bin/python3 -m pip install --upgrade pip
# Install the required dependencies
./env/bin/python3 -m pip install -r requirements_librarian.txt
# Deactivate the virtual environment
deactivate
conjurer_librarian/requirements_librarian.txt
+1 -1
View File
@@ -1,3 +1,3 @@
habanero
waitress
flask
flask[async]
conjurer_librarian/scrape_bot.py
+74 -33
View File
@@ -1,46 +1,67 @@
"""
This module contains the code for the scrape bot.
"""
import json
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from requests import Timeout,ConnectTimeout, ConnectionError
from queue import Queue
import logging
import random
import re
import time
import random
import logging
from json import JSONDecodeError
from queue import Queue
from threading import Thread
SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
from urllib.request import urlopen
from requests import ConnectionError as RequestsConnectionError
from requests import ConnectTimeout, Timeout
SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
SCR_FILENAME = "40_chunk.txt"
SCR_ENCODING = "utf-8"
WORK_Q = Queue()
random.seed()
def load_ndb_to_q(logger):
"""
This function loads items from the not_in_db.json file into the work queue.
"""
logger.info("Loader started")
while True:
while True:
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
for item in ndb_database.keys():
logger.info(item)
url = f"https://sci-hub.se/{item}"
ndb_database = json.load(ndb_file)
for _ in range (1,10):
key = next(iter(ndb_database))
item = ndb_database.pop(key)
logger.info(key)
url = f"https://sci-hub.se/{key}"
WORK_Q.put([item, url, False])
ndb_file.truncate(0)
ndb_file.seek(0)
json.dump(ndb_database, ndb_file, indent=4)
except JSONDecodeError:
time.sleep(60*60*2)
pass
time.sleep(180)
time.sleep(60 * 60 * 3)
time.sleep(60*60*3)
def check_if_exists_brute_force(logger):
"""
This function checks if a document exists using brute force search.
"""
# Function code here
while True:
logger.info("Scraper tick")
item = WORK_Q.get()
page_url =item[1]
page_url = item[1]
logger.error("REFINE: Brute force search!")
if not page_url.startswith(("http:", "https:")):
raise ValueError("URL must start with 'http:' or 'https:'")
blocked = True
try:
# trunk-ignore(bandit/B310)
# trunk-ignore(bandit/B310)
with urlopen(page_url) as response:
data = response.read()
text = data.decode("utf-8")
@@ -50,44 +71,64 @@ def check_if_exists_brute_force(logger):
blocked = False
else:
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
if re.match(
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
line,
):
blocked = False
logger.info("Not found")
item[2] = False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
if m := re.match(
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
):
blocked = False
direct_download_link = "https:" + str(m.group(1))
logger.info (direct_download_link)
with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
logger.info(direct_download_link)
with open(
SCR_DATABASE_PATH + SCR_FILENAME,
"a",
encoding=SCR_ENCODING,
) as operated_file:
operated_file.write("\n")
operated_file.write(item[0])
item[2] = True
except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
except (
Timeout,
ConnectTimeout,
ConnectionRefusedError,
ConnectionError,
RequestsConnectionError,
):
pass
if blocked:
logger.info(item)
logger.info(text)
logger.error("Got blocked. Fuck.")
time.sleep(60*60*72)
time.sleep(60 * 60 * 72)
# trunk-ignore(bandit/B311)
rand = random.randint(1,100)
logger.info(f"Sleeping for {3*rand} minutes")
time.sleep(180*rand)
rand = random.randint(1, 60)
logger.info(f"Sleeping for {2*rand} minutes")
time.sleep(120 * rand)
def scraper(logger = None):
def scraper(logger=None):
"""
This function is responsible for scraping data.
"""
if not logger:
logger = logging.getLogger()
logger.setLevel("DEBUG")
h1 = logging.StreamHandler()
logger.addHandler(h1)
logger.info("No logger set. Test run")
loader = Thread(target = load_ndb_to_q, args = (logger,))
scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
loader = Thread(target=load_ndb_to_q, args=(logger,))
_scraper = Thread(target=check_if_exists_brute_force, args=(logger,))
loader.start()
scraper.start()
_scraper.start()
loader.join()
scraper.join()
_scraper.join()
if __name__ == "__main__":
scraper()
scraper()
conjurer_librarian/search_bot.py
+123 -58
View File
@@ -1,26 +1,55 @@
#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
"""
This module contains functions for searching for DOI (Digital Object Identifier)
in a list of live results.
It includes a producer-consumer pattern implementation using threads and queues.
from queue import Queue, Empty
Functions:
- producer: Reads lines from a file and puts them into an output queue.
- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
Global Variables:
- MAXTHREADS: Maximum number of worker threads.
- DATABASE_PATH: Path to the database files.
- ENCODING: Encoding of the database files.
- CHUNK: File name pattern for the database files.
- _sentinel: Sentinel object used to signal termination.
- result_list: List to store the search results.
- WORK_Q_SIZE: Maximum size of the work queue.
"""
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
from queue import Empty, Queue
from threading import Thread
q = Queue()
#TODO: DATA FOR TEST ONLY
#MAXTHREADS = 5
#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
# DATA FOR TEST ONLY
# MAXTHREADS = 5
# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
#TODO: DEPLOYMENT DATA
# DEPLOYMENT DATA
MAXTHREADS = 41
DATABASE_PATH = r'C:\\Database\\chunks\\'
DATABASE_PATH = r"C:\\Database\\chunks\\"
ENCODING = "utf-8"
CHUNk = "_chunk.txt"
CHUNK = "_chunk.txt"
_sentinel = object()
result_list = []
WORK_Q_SIZE = 15500000
WORK_Q_SIZE = 35500000
def producer(out_q, control_q, filename, logger):
def producer(out_q, control_q, filename, _logger):
"""
Produces items from the output queue and puts them into the control queue.
Args:
out_q (Queue): Output queue.
control_q (Queue): Control queue.
filename (str): Name of the file.
_logger: Logger object for logging.
"""
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
logger.info(f"Worker {filename} ")
_logger.info(f"Worker {filename} ")
while True:
line = operated_file.readline()
if not line:
@@ -30,77 +59,113 @@ def producer(out_q, control_q, filename, logger):
check = control_q.get(block=False)
except Empty:
check = False
pass
if check is _sentinel:
logger.info("TERM signal received")
_logger.info("TERM signal received")
control_q.put(check)
break
logger.info(f"Worker finished: {filename}")
_logger.info(f"Worker finished: {filename}")
out_q.put(_sentinel)
#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
def consumer(in_q, control_q, doi, live_results, logger):
sentinels = 0
# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
"""
Consumes items from an input queue and checks if DOI exists in the live results list.
Args:
in_q (Queue): Input queue.
control_q (Queue): Control queue.
doi (list): List of DOI to search for.
live_results (list): List to store the search results.
_logger: Logger object for logging.
"""
_logger.info("Consumer thread started:%s", no)
for item in doi:
result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
while True:
done_check = True
data = in_q.get()
if data is _sentinel:
logger.info("Worker finished")
sentinels += 1
else:
for item in result_list:
if item["DOI"] in data and not item["exists"]:
logger.info(data)
logger.info("HIT")
item["exists"] = True
live_results.append(item)
done_check = done_check and item["exists"]
if done_check:
control_q.put(_sentinel)
if sentinels >= MAXTHREADS:
logger.info("All workers finished")
try:
data = in_q.get(block=False)
if data is _sentinel:
_logger.info("Worker finished")
control_dict["sentinels"] += 1
else:
for item in result_list:
if item["DOI"] in data and not item["exists"]:
_logger.info(data)
_logger.info("HIT")
item["exists"] = True
live_results.append(item)
done_check = done_check and item["exists"]
if done_check:
control_q.put(_sentinel)
except Empty:
pass
if control_dict["sentinels"] >= MAXTHREADS:
_logger.info("All workers finished")
break
def search_for_doi(doi, live_results, logger):
def search_for_doi(doi, live_results, _logger):
"""
Search for DOI in live_results using _logger for logging.
Args:
doi (list): List of DOI to search for.
live_results (list): List to store the search results.
_logger: Logger object for logging.
"""
control_dict = {"sentinels":0}
result_list = []
threads = []
work_q = Queue(maxsize=WORK_Q_SIZE)
control_q = Queue()
t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
logger.info("Consumer thread created")
threads.append(t_cons)
for i in range(0,MAXTHREADS):
#TODO: TEST DATA
#filename = "test" + str(i) + CHUNk
#TODO: DEPLOYMENT DATA
filename = str(i) + CHUNk
logger.info(f"Creating worker thread no: {i}")
threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
for i in range (0, (len(doi)//1000)+2):
t_cons = Thread(
target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)
)
_logger.info("Consumer thread created")
threads.append(t_cons)
for i in range(0, MAXTHREADS):
# TEST DATA
# filename = "test" + str(i) + CHUNk
# DEPLOYMENT DATA
filename = str(i) + CHUNK
_logger.info(f"Creating worker thread no: {i}")
threads.append(
Thread(target=producer, args=(work_q, control_q, filename, _logger))
)
for worker in threads:
worker.start()
for worker in threads:
worker.join()
return result_list
if __name__ == "__main__":
import logging
logger = logging.getLogger()
logger.setLevel("DEBUG")
h1 = logging.StreamHandler()
logger.addHandler(h1)
logger.info("TEST RUN")
live_result = []
logger.info(search_for_doi([
("10.1002/9781118786352.wbieg0998.pub2","DATA"),
("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
("10.2165/00128415-200309690-00017","DATA"),
("10.5772/48313","DATA"),
("10.15803/ijnc.7.2_419","DATA"),
("10.1051/0004-6361/201321596e","DATA"),
("10.2307/40835941","DATA"),
], live_result, logger))
logger.info(
search_for_doi(
[
("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
("10.2165/00128415-200309690-00017", "DATA"),
("10.5772/48313", "DATA"),
("10.15803/ijnc.7.2_419", "DATA"),
("10.1051/0004-6361/201321596e", "DATA"),
("10.2307/40835941", "DATA"),
],
live_result,
logger,
)
)
logger.info(live_result)
conjurer_librarian/test_send.py
+45
View File
@@ -0,0 +1,45 @@
import asyncio
import json
import logging
from json.decoder import JSONDecodeError
from logging import handlers
import requests
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
async def send_results():
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
h1 = handlers.RotatingFileHandler(
filename="D:\\logs\\librarian.log",
encoding="utf-8",
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
)
logger.addHandler(h1)
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
except JSONDecodeError:
pass
result = database["4bacbca0-ce3b-45c6-90a1-05c70b46b740"]
result = {"4bacbca0-ce3b-45c6-90a1-05c70b46b740":result}
coroutine = asyncio.to_thread(
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
logger.info("SENT")
result = await coroutine
logger.info(result.status_code)
logger.info("SEND CONFIRMED")
async def main():
await send_results()
if __name__ == "__main__":
asyncio.run(main())
conjurer_musician/conjurer_musician.py
+118 -74
View File
@@ -35,34 +35,11 @@ if platform in ("linux", "linux2"):
LOGSTORE = "/home/pi/RetroPie/logs/"
ENCODING = "utf-8"
MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/"
music_file_list = []
def create_playlist():
"""
Reads a JSON file containing a playlist and writes the playlist items to a text file.
The JSON file path is '/home/pi/Conjurer/playlist.json',
and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
Raises:
JSONDecodeError: If the JSON file cannot be decoded.
"""
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
with open(
"/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
) as w_file:
try:
playlist = json.load(r_file)
for item in playlist:
w_file.write(item)
w_file.write("\n")
except json.JSONDecodeError:
pass
priority_list = []
def rescan():
"""
@@ -70,18 +47,35 @@ def rescan():
and adds them to a list of music files.
"""
logging.info("Rescan triggered")
for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
temp_music_file = mp3_item.as_posix()
if platform == "win32":
temp_music_file = temp_music_file.replace("/", "\\")
music_file_list.append(temp_music_file)
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
database = music_file_list
# self.app.logger.info("DUMPING DATA")
s_file.truncate(0)
s_file.seek(0)
json.dump(database, s_file)
create_playlist()
for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"):
temp_music_file = mp3_item.as_posix()
if platform == "win32":
temp_music_file = temp_music_file.replace("/", "\\")
priority_list.append(temp_music_file)
with open(
"/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
) as w_file:
try:
for item in music_file_list:
w_file.write(item)
w_file.write("\n")
except json.JSONDecodeError:
pass
with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
try:
for item in priority_list:
w_file.write(item)
w_file.write("\n")
except json.JSONDecodeError:
pass
def thread_rescan():
@@ -202,7 +196,9 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
return_list.append(search_weight[itr])
if not return_to_bot:
with open(
"/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
"/home/pi/Conjurer/priority_queue.playlist",
"r+",
encoding="utf-8",
) as s_file:
s_file.write(search_weight[itr][1])
break
@@ -211,13 +207,6 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
fun_logger.info("Wiele plików do zagrania")
search_weight.sort(key=lambda x: x[0], reverse=True)
return_list.extend(search_weight[: int(how_many)])
if not return_to_bot:
with open(
"/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8"
) as s_file:
for item in return_list:
s_file.write(item[1] + "\n")
return return_list
@@ -252,24 +241,36 @@ def remove_characters(string, character):
@app.route("/stream", methods=["GET"])
def stream_music():
"""
The function `get_music_list` returns a JSON object containing a list of music files.
:return: A JSON response containing a key "music_file_list" with the value of the variable
`music_file_list`.
The function `stream_music` is a route handler for the "/stream" endpoint.
It returns a JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
:return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
"""
return send_from_directory("/tmp/hls", "stream.m3u8")
@app.route('/<string:file_name>')
@app.route("/<string:file_name>")
def stream(file_name):
video_dir = '/tmp/hls'
"""
Stream the specified file from the video directory.
Args:
file_name (str): The name of the file to be streamed.
Returns:
Response: The response containing the streamed file.
"""
video_dir = "/tmp/hls"
return send_from_directory(video_dir, file_name)
@app.route("/stream_mp3", methods=["GET"])
def stream_music_mp3():
"""
The function `get_music_list` returns a JSON object containing a list of music files.
:return: A JSON response containing a key "music_file_list" with the value of the variable
`music_file_list`.
The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint.
It redirects the user to the URL "http://www.example.com" with a status code of 302.
:return: A redirect response to "http://www.example.com" with a status code of 302.
"""
return redirect("http://www.example.com", code=302)
@@ -277,19 +278,18 @@ def stream_music_mp3():
@app.route("/clear_pr_pls", methods=["GET"])
def clear_pr_pls():
"""
The function `get_music_list` returns a JSON object containing a list of music files.
:return: A JSON response containing a key "music_file_list" with the value of the variable
`music_file_list`.
The function `clear_pr_pls` clears the contents of the priority queue playlist file.
:return: A JSON response indicating the success of the operation.
"""
with open(
"/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
) as s_file:
s_file.truncate(0)
app.logger.info("CLEARING PLAYLIST")
with open('/home/pi/Conjurer/priority_queue.playlist', 'w', encoding='utf-8') as cleared_pl:
cleared_pl.write('')
return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
return return_data, 200
@app.route("/mp3", methods=["GET"])
def get_music_list():
"""
@@ -324,20 +324,22 @@ def update_music_list():
@app.route("/get_music", methods=["POST"])
def look_for_playlist():
"""
The function `update_music_list` receives a POST request with a JSON payload, logs the received
The function `look_for_playlist` receives a POST request with a JSON payload, logs the received
item, adds it to a music file list, and returns a success message along with the updated record.
:return: The function `update_music_list` is returning a tuple containing a JSON response and a
status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
with values indicating the success of the operation and the data that was
received and added to the `music_file_list`.
The status code returned is 200, indicating a successful response.
:return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`,
`message`, `statusCode`, and `data`, with values indicating the success of the operation and the
data that was received and added to the `music_file_list`. The status code returned is 200,
indicating a successful response.
"""
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["lista_slow"])
app.logger.info(record["UUID"])
app.logger.info(record["dlugosc_plejlisty"])
return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True)
return_data = wyszukaj(
record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True
)
return_data = (
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
@@ -345,26 +347,58 @@ def look_for_playlist():
)
return return_data
@app.route("/request_radio_file", methods=["POST"])
def add_request():
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["lista_slow"])
app.logger.info(record["UUID"])
return_data = wyszukaj(
record["lista_slow"], 0, app.logger, False
)
with open(
"/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
) as s_file:
for item in return_data:
s_file.write(item[1] + "\n")
return_data = (
jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
200,
)
return return_data
@app.route("/add_to_priority", methods=["POST"])
def add_to_priority():
"""
The function `update_music_list` receives a POST request with a JSON payload, logs the received
The function `add_to_priority` receives a POST request with a JSON payload, logs the received
item, adds it to a music file list, and returns a success message along with the updated record.
:return: The function `update_music_list` is returning a tuple containing a JSON response and a
status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
with values indicating the success of the operation and the data that was
received and added to the `music_file_list`.
The status code returned is 200, indicating a successful response.
:return: A tuple containing a JSON response and a status code.
The JSON response includes keys `isError`,
`message`, `statusCode`, and `data`, with values
indicating the success of the operation and the
data that was received and added to the `music_file_list`.
The status code returned is 200,indicating a successful response.
"""
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["lista_slow"])
app.logger.info(record["UUID"])
app.logger.info(record["dlugosc_plejlisty"])
return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False)
return_data = wyszukaj(
record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
)
with open(
"/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8"
) as s_file:
for item in return_data:
s_file.write(item[1] + "\n")
return_data = (
jsonify(isError=False, message="Success", statusCode=200, data=return_data),
jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
200,
)
return return_data
@@ -381,8 +415,18 @@ def flask_debug():
def waitress_run():
"""
The `waitress_run` function serves the `app` on host "0.0.0.0"
The `waitress_run` function serves the `app` on host HOST_ADDRESS
and port 5000 using the Waitress WSGI server.
This function starts the Waitress server and listens for incoming requests
on the specified host and port. It uses the `serve` function from the Waitress
library to handle the serving process.
Parameters:
None
Returns:
None
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
@@ -406,7 +450,7 @@ if __name__ == "__main__":
rescan()
logger.info("Started")
threads = []
#threads.append(threading.Thread(target=flask_debug))
# threads.append(threading.Thread(target=flask_debug))
threads.append(threading.Thread(target=waitress_run))
threads.append(threading.Thread(target=thread_rescan))
conjurer_musician/radio_conjurer.liq
+135 -43
View File
@@ -1,48 +1,140 @@
# FILEPATH: /home/mtuszowski/conjurer/conjurer_musician/radio_conjurer.liq
# This script sets up a Liquidsoap radio stream with various features and configurations.
# Load icecast credentials from a JSON file
let json.parse credentials = file.contents("/home/pi/Conjurer/icecast_credentials.json")
enable_replaygain_metadata()
s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist"))
s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist"))
s = random(id="randomizer", weights=[7, 10], [s2, s1])
interactive.harbor(port = 9999)
f = interactive.float("f", description="Frequency", min=0., max=1000.,unit="Hz", 200.)
g = interactive.float("g", description="Gain", min=0., max=20.,unit="dB", 8.)
b = bass_boost(frequency=f, gain=g, s)
s = add([s, b])
a = interactive.float("main_volume", min=0., max=20., 1.)
s = compress.multiband.interactive(bands=3, s)
s = nrj(normalize (s))
s = crossfade(fade_out=3., fade_in=3., duration=5., s)
s = amplify(a,s)
s = blank.skip(max_blank=2., s)
log_to_stdout = true
log_to_file = true
logpath = "/home/pi/Conjurer/radio_log.log"
loglevel = 4
set("log.stdout",log_to_stdout)
set("log.level",loglevel)
# Enable logging to file. See variable: logpath
set("log.file",log_to_file)
set("log.file.path",logpath)
emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
radio = fallback(id="switcher",track_sensitive = false, [s, emergency])
handle_metadata = fun (m) -> begin
log.info(m["title"])
log.info(m["artist"])
log.info(m["filename"])
end
radio.on_metadata(handle_metadata)
interactive.persistent("script.params")
# Enable replaygain metadata processing
enable_replaygain_metadata()
l = playlog(duration = 14400.0, persistency="/home/pi/Conjurer/persistence.log")
def check(r)
m = request.metadata(r)
if l.last(m) < 3600. then
log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).")
false
else
l.add(m)
true
end
end
# Enable logging on Standard Output and set logging level.
output.file.hls("/tmp/hls",[("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))],radio)
output.icecast(%mp3, host="retropie", port=8000,password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
# Define playlists to be used in the stream
s1 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/all_playlist.playlist"))
s2 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/priority_queue.playlist"))
s3 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/hit.playlist"))
requests_queue = request.queue()
def queue_processing()
f1=file.read("/home/pi/Conjurer/request.playlist")
text = f1()
if text != "" then
requests_queue.push.uri(text)
file.remove("/home/pi/Conjurer/request.playlist")
f = file.open("/home/pi/Conjurer/request.playlist", create=true)
f.close()
log.info("done")
else
log.info("no requests")
end
end
# Randomly select a playlist with weights
s = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3])
#s = delay(duration=5., s)
#radio = add([delay(1800., jingles), music])
#radio = fallback([delay(1800., jingles), music])
#p = predicate.signal()
#radio = switch([(p, jingles), ({true}, music)])
#thread.run(every=1200., {p.signal()})
# def cmd(_)
# p.signal()
# "Jingle inserted!"
# end
# server.register("insert_jingle", cmd)
# Set up an interactive harbor for controlling the stream
interactive.harbor(port = 9999)
# Set up interactive controls for bass boost
f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.)
g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.)
b = bass_boost(frequency=f, gain=g, s)
s = add([s, b])
# Set up interactive control for main volume
a = interactive.float("main_volume", min=0., max=20., 1.)
s = compress.multiband.interactive(bands=3, s)
# Apply audio processing effects
s = nrj(normalize(s))
s = amplify(a, s)
# Skip blank sections in the stream
s = blank.skip(max_blank=2., s)
# Configure logging settings
log_to_stdout = true
log_to_file = true
logpath = "/home/pi/Conjurer/radio_log.log"
loglevel = 4
set("log.stdout", log_to_stdout)
set("log.level", loglevel)
# Enable logging to file
set("log.file", log_to_file)
set("log.file.path", logpath)
# Set up emergency fallback track
emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
# Create a fallback stream with the main stream and emergency track
cross_s = crossfade(fade_out=3., fade_in=3., duration=5., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
radio = fallback(id="switcher2", track_sensitive=false, [cross_s, emergency])
thread.run(every=60., queue_processing)
# Handle metadata events
handle_metadata = fun (m) -> begin
log.info(m["title"])
log.info(m["artist"])
log.info(m["filename"])
end
radio.on_metadata(handle_metadata)
# Enable persistent script parameters
interactive.persistent("script.params")
# Configure output formats and destinations
output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
# output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio)
# output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio)
# output.icecast(%aac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="aac-stream", radio)
# output.icecast(%flac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="flac-stream", radio)
# output.icecast(%vorbis, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="vorbis-stream", radio)
# output.icecast(%speex, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="speex-stream", radio)
# output.icecast(%wav, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="wav-stream", radio)
# output.icecast(%pcm, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="pcm-stream", radio)
# output.icecast(%raw, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="raw-stream", radio)
# output.icecast(%s16l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16l-stream", radio)
# output.icecast(%s16b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16b-stream", radio)
# output.icecast(%s24l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24l-stream", radio)
# output.icecast(%s24b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24b-stream", radio)
# output.icecast(%s32l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32l-stream", radio)
# output.icecast(%s32b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32b-stream", radio)
# output.icecast(%s64l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64l-stream", radio)
# output.icecast(%s64b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64b-stream", radio)
# output.icecast(%s128l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128l-stream", radio)
# output.icecast(%s128b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128b-stream", radio)
# output.icecast(%s256l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256l-stream", radio)
# output.icecast(%s256b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256b-stream", radio)
# output.icecast(%s512l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512l-stream", radio)
# output.icecast(%s512b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512b-stream", radio)
# output.icecast(%s1024l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024l-stream", radio)
# output.icecast(%s1024b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024b-stream", radio)
# output.icecast(%s2048l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048l-stream", radio)
# output.icecast(%s2048b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048b-stream", radio)
# output.icecast(%s4096l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096l-stream", radio)
# output.icecast(%s4096b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096b-stream", radio)
# output.icecast(%s8192l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192l-stream", radio)
# output.icecast(%s8192b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192b-stream", radio)
conjurer_musician/requirements_musician.txt
+2 -1
View File
@@ -1,2 +1,3 @@
flask
waitress
waitress
sshtunnel
cr_results.json
View File
install_music_bot.sh
+9 -1
View File
@@ -8,9 +8,17 @@ cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjur
cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
touch /home/pi/Conjurer/discord_mus_service.log
touch /home/pi/Conjurer/all_playlist.txt
touch /home/pi/Conjurer/playlist.json
touch /home/pi/Conjurer/prio_playlist.json
touch /home/pi/Conjurer/all_playlist.playlist
touch /home/pi/Conjurer/priority_queue.playlist
touch /home/pi/Conjurer/hit.playlist
touch /home/pi/Conjurer/request.playlist
touch /home/pi/Conjurer/persistence.log
echo "[]" > /home/pi/Conjurer/persistence.log
touch /home/pi/Conjurer/radio_log.log
./env/bin/python3 -m pip install --upgrade pip
./env/bin/python3 -m pip install -r requirements_musician.txt
not_in_db.json
View File
rr_results.json
View File
s_results.json
View File
settings.json
+7
View File
@@ -204,6 +204,13 @@
false,
false
],
"same plusy\\b": [
"Jak na cmentarzu Szefie. Jak na cmentarzu",
15.0,
0,
false,
false
],
"kld\\b": [
"KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.",
15.0,
spotify_dl/spotify.py
+2
View File
@@ -239,6 +239,8 @@ def get_item_name(sp, item_type, item_id):
name = sp.album(album_id=item_id).get("name")
elif item_type == "track":
name = sp.track(track_id=item_id).get("name")
elif item_type == "artist_top_tracks":
name = sp.artist_top_tracks(artist_id=item_id).get("name")
return sanitize(name)