Compare commits

..

27 Commits

Author SHA1 Message Date
gitea 0e40741288 FIx of volume 2024-04-23 12:30:22 +02:00
gitea 253122e671 Fix fix fix 2024-04-23 12:28:19 +02:00
gitea c0a0389d46 FIX! 2024-04-23 12:27:40 +02:00
gitea 14693d9015 Maybe fix ? 2024-04-23 12:27:02 +02:00
gitea 897a9c1313 test 2024-04-23 12:24:00 +02:00
gitea 96a106b477 el fixo stupido 2024-04-23 12:13:58 +02:00
gitea 37df223233 test 2024-04-23 12:12:22 +02:00
gitea 3cd6a850c4 Fix fix fix 2024-04-23 12:01:10 +02:00
gitea 74f1c5dead sx 2024-04-23 11:58:31 +02:00
gitea b5caed0622 Small fix 2024-04-23 11:57:52 +02:00
gitea 8cde1e741c Clearing fetch failed 2024-04-23 11:56:35 +02:00
gitea df347b71d3 Change log to print 2024-04-23 11:54:44 +02:00
gitea dc14794421 Fixes 2024-04-23 11:46:17 +02:00
gitea 7383391902 Fix 2024-04-23 11:16:50 +02:00
gitea 59558ba1af Radio hardkor 2024-04-23 10:42:19 +02:00
gitea 3b8833cd3b Radio requests 2024-04-23 08:56:34 +01:00
gitea c043726ab2 Fix 2024-04-22 20:19:31 +02:00
gitea e44bc7f64a Fixes 2024-04-22 20:19:31 +02:00
gitea 8f006b4323 Requests 2024-04-22 20:19:30 +02:00
gitea 97f0c7d33d Mostly working ? 2024-04-22 20:19:30 +02:00
gitea 1a75f5588d Fix fix 2024-04-22 20:19:30 +02:00
gitea 430a66c3ba Fixes! 2024-04-22 20:19:30 +02:00
gitea 4d8218003d Testing of new functions in conjurer_musician.py and radio_conjurer.liq 2024-04-22 20:19:30 +02:00
gitea fbecb7f82e Tmp chng 2024-04-22 20:19:30 +02:00
gitea 0134ce240f First step towards cogization 2024-04-22 20:19:30 +02:00
gitea 4cc828373b SMall fixes for tunnel 2024-04-22 20:19:30 +02:00
gitea a521724857 New requirement 2024-04-22 20:19:30 +02:00
21 changed files with 336 additions and 1056 deletions
-7
View File
@@ -1,7 +0,0 @@
enable=all
source-path=SCRIPTDIR
disable=SC2154
# If you're having issues with shellcheck following source, disable the errors via:
# disable=SC1090
# disable=SC1091
+11 -15
View File
@@ -2,38 +2,34 @@
# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
version: 0.1 version: 0.1
cli: cli:
version: 1.22.2 version: 1.20.1
# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
plugins: plugins:
sources: sources:
- id: trunk - id: trunk
ref: v1.6.0 ref: v1.4.5
uri: https://github.com/trunk-io/plugins uri: https://github.com/trunk-io/plugins
# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
runtimes: runtimes:
enabled: enabled:
- go@1.21.0
- node@18.12.1 - node@18.12.1
- python@3.10.8 - python@3.10.8
# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
lint: lint:
enabled: enabled:
- shellcheck@0.10.0 - actionlint@1.6.27
- shfmt@3.6.0 - bandit@1.7.8
- taplo@0.8.1 - black@24.3.0
- actionlint@1.7.1
- bandit@1.7.9
- black@24.4.2
- codespell - codespell
- checkov@3.2.139 - checkov@3.2.48
- git-diff-check - git-diff-check
- isort@5.13.2 - isort@5.13.2
- pylint - pylint
- markdownlint@0.41.0 - markdownlint@0.39.0
- prettier@3.3.2 - prettier@3.2.5
- ruff@0.4.9 - ruff@0.3.4
- trivy@0.52.2 - trivy@0.50.1
- trufflehog@3.78.1 - trufflehog@3.71.0
- yamllint@1.35.1 - yamllint@1.35.1
actions: actions:
disabled: disabled:
+87 -258
View File
@@ -16,7 +16,6 @@ import os
import random import random
import re import re
import shutil import shutil
import subprocess
import sys import sys
import threading import threading
import uuid import uuid
@@ -42,16 +41,11 @@ from discord.ext import commands, tasks
from spotipy.oauth2 import SpotifyClientCredentials from spotipy.oauth2 import SpotifyClientCredentials
import yt_dlp import yt_dlp
from communication_subroutine import (
IN_COMM_Q,
OUT_COMM_Q,
QueryControl,
comm_subroutine,
PREPPED_TRACKS,
)
from spotify_dl import spotify from spotify_dl import spotify
from spotify_dl import youtube as youtube_download from spotify_dl import youtube as youtube_download
from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q, QueryControl
# *=========================== Classes definition # *=========================== Classes definition
# The above code is defining a TypedDict called `Music_Config` in Python. This TypedDict has three # The above code is defining a TypedDict called `Music_Config` in Python. This TypedDict has three
@@ -68,6 +62,7 @@ Music_Config = TypedDict(
) )
class MusicFileList(object): class MusicFileList(object):
""" """
The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a
@@ -140,15 +135,10 @@ MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15
MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30 MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000" FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321"
SKIP_TRACK = "/skip"
GET_MP3 = "/mp3" GET_MP3 = "/mp3"
SEND_MP3 = "/update_mp3" SEND_MP3 = "/update_mp3"
GET_PLAYLIST = "/get_music" GET_PLAYLIST = "/get_music"
ADD_TO_PRIO_PLAYLIST = "/add_to_priority" ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
REQUEST_MUSIC = "/request_radio_file" REQUEST_MUSIC = "/request_radio_file"
CLEAR_PRIO = "/clear_pr_pls" CLEAR_PRIO = "/clear_pr_pls"
LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001" LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
@@ -288,6 +278,7 @@ for key in word_reactions:
logger.info("Done") logger.info("Done")
# *=========================================== Create Client # *=========================================== Create Client
logger.info("Creating discord bot") logger.info("Creating discord bot")
client = commands.Bot(intents=intents, command_prefix="$") client = commands.Bot(intents=intents, command_prefix="$")
@@ -407,13 +398,7 @@ async def on_message(message):
result, MESSAGE_TABLE = await handle_response( result, MESSAGE_TABLE = await handle_response(
prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION" prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION"
) )
await message.reply(result)
if len(result) < 1500:
await message.reply(result)
else:
while len(result) > 1500:
await message.reply(result[:1500])
result = result[1500:]
if "imaginuje sobie:" in message.content: if "imaginuje sobie:" in message.content:
async with channel.typing(): async with channel.typing():
logger.info("Poczatek procedury obrazkowej") logger.info("Poczatek procedury obrazkowej")
@@ -646,12 +631,7 @@ async def play(ctx, zamawial=None, arg=None):
query, vykidailo, bartender, message_table_muzyka, username, "MUSIC" query, vykidailo, bartender, message_table_muzyka, username, "MUSIC"
) )
logger.debug("Obecna historia czatu: %s", message_table_muzyka) logger.debug("Obecna historia czatu: %s", message_table_muzyka)
if len(result) < 1500: await ctx.send(result)
await ctx.send(result)
else:
while len(result) > 1500:
await ctx.send(result[:1500])
result = result[1500:]
async def archive_channel(channel_no): async def archive_channel(channel_no):
@@ -690,7 +670,6 @@ async def check_music():
await play(MUZYKA["ctx"], MUZYKA["requester"]) await play(MUZYKA["ctx"], MUZYKA["requester"])
await asyncio.sleep(2) await asyncio.sleep(2)
@tasks.loop(seconds=3) @tasks.loop(seconds=3)
async def check_data_q(): async def check_data_q():
""" """
@@ -706,42 +685,39 @@ async def check_data_q():
for doi in fresh_data.entries: for doi in fresh_data.entries:
logger.info(doi) logger.info(doi)
desc = fresh_data.entries[doi] desc = fresh_data.entries[doi]
title = desc["Title"][0] title = desc['Title'][0]
entries.append( entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n")
f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n" l_p+=1
) message = '*Z podłogi wysuwa się winda na książki*'
l_p += 1
message = "*Z podłogi wysuwa się winda na książki*"
if fresh_data.ctx is not None: if fresh_data.ctx is not None:
ctx = fresh_data.ctx ctx = fresh_data.ctx
message += f" Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}" message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}'
else: else:
ctx = client.get_channel(1062047571557744721) ctx = client.get_channel(1062047367337095268)
message += f" Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał" message += f' Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał'
message += "nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku "
if len(entries) < 1: message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
if len(entries)<1:
message += "niestety nie ma nic" message += "niestety nie ma nic"
await ctx.send(message) elif len(entries)>=1 and len(entries)<5:
elif len(entries) >= 1 and len(entries) < 5:
message += " znajduje się coś:\n" message += " znajduje się coś:\n"
for item in entries: for item in entries:
message += item message+=item
await ctx.send(message)
else: else:
message += " znajduje się cholernie dużo:\n" message+= " znajduje się cholernie dużo:\n"
for item in entries: for item in entries:
message += item message+=item
if len(message) > 1500: if len(message) > 1500:
await ctx.send(message) await ctx.send(message)
message = "" message = ""
# Kept for sentimental reasons await ctx.send(message)
# await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}") #Kept for sentimental reasons
#await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
except Empty: except Empty:
pass pass
@tasks.loop(seconds=120) @tasks.loop(seconds=120)
async def check_self(): async def check_self():
""" """
@@ -952,17 +928,17 @@ async def handle_response(
"Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size "Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size
) )
if request_type == "MUSIC": if request_type == "MUSIC":
algorithm = "gpt-4o" algorithm = "gpt-3.5-turbo-0125"
table = message_table_muzyka table = message_table_muzyka
token_amount = 10700 token_amount = 10700
elif request_type == "RANDOM": elif request_type == "RANDOM":
algorithm = "gpt-4o" algorithm = "gpt-3.5-turbo-0125"
table = MESSAGE_TABLE table = MESSAGE_TABLE
token_amount = 10700 token_amount = 10700
else: else:
table = MESSAGE_TABLE table = MESSAGE_TABLE
algorithm = "gpt-4o" algorithm = "gpt-4"
token_amount = 10700 token_amount = 7000
prompt_gpt_request_size = num_tokens_from_string( prompt_gpt_request_size = num_tokens_from_string(
{"role": "user", "content": final_prompt}, "gpt-4" {"role": "user", "content": final_prompt}, "gpt-4"
@@ -1044,33 +1020,26 @@ async def handle_response(
logger.info(result) logger.info(result)
temp = {"role": "assistant", "content": result} temp = {"role": "assistant", "content": result}
history.append(temp) history.append(temp)
if request_type == "MUSIC": if request_type:
with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file_music_memory: with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file:
# First we load existing data into a dict. # First we load existing data into a dict.
file_data = json.load(file_music_memory) file_data = json.load(file)
# Join new_data with file_data inside emp_details # Join new_data with file_data inside emp_details
file_data.append(temp) file_data.append(temp)
file_music_memory.seek(0) # Sets file's current position at offset.
file.seek(0)
# convert back to json. # convert back to json.
json.dump(file_data, file_music_memory, indent=4) json.dump(file_data, file, indent=4)
elif request_type == "RANDOM":
with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
# First we load existing data into a dict.
file_data = json.load(file_memory)
# Join new_data with file_data inside emp_details
file_data.append(temp)
file_memory.seek(0)
# convert back to json.
json.dump(file_data, file_memory, indent=4)
else: else:
with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory: with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file:
# First we load existing data into a dict. # First we load existing data into a dict.
file_data = json.load(file_memory) file_data = json.load(file)
# Join new_data with file_data inside emp_details # Join new_data with file_data inside emp_details
file_data.append(temp) file_data.append(temp)
file_memory.seek(0) # Sets file's current position at offset.
file.seek(0)
# convert back to json. # convert back to json.
json.dump(file_data, file_memory, indent=4) json.dump(file_data, file, indent=4)
return result, MESSAGE_TABLE return result, MESSAGE_TABLE
@@ -1095,19 +1064,11 @@ async def get_file(ctx, source, link):
if source == "Spotify": if source == "Spotify":
dir_path = None dir_path = None
item_type, item_id = spotify.parse_spotify_url(link) item_type, item_id = spotify.parse_spotify_url(link)
if item_type in ["album", "track", "playlist"] and item_id: if item_id:
logger.info(item_type) coro = asyncio.to_thread(spotify.fetch_tracks, spotify_ctrl, item_type, link)
logger.info(item_id) file_list = await coro
logger.info("Spotify link provided") coro = asyncio.to_thread(spotify.get_item_name, spotify_ctrl, item_type, item_id)
file_list = await asyncio.to_thread( directory_name = await coro
spotify.fetch_tracks, spotify_ctrl, item_type, link
)
logger.info(file_list)
directory_name = await asyncio.to_thread(
spotify.get_item_name, spotify_ctrl, item_type, item_id
)
logger.info(directory_name)
logger.info("Spotify scrape done")
url_data = {"urls": []} url_data = {"urls": []}
url_dict = {} url_dict = {}
url_dict["save_path"] = Path( url_dict["save_path"] = Path(
@@ -1117,7 +1078,6 @@ async def get_file(ctx, source, link):
url_dict["songs"] = file_list url_dict["songs"] = file_list
url_data["urls"].append(url_dict.copy()) url_data["urls"].append(url_dict.copy())
file_name_f = youtube_download.default_filename file_name_f = youtube_download.default_filename
logger.info("YT-DL")
coro = asyncio.to_thread( coro = asyncio.to_thread(
youtube_download.download_songs, youtube_download.download_songs,
songs=url_data, songs=url_data,
@@ -1133,7 +1093,6 @@ async def get_file(ctx, source, link):
proxy="", proxy="",
) )
_ = await coro _ = await coro
logger.info("YT-DL done")
dir_path = (url_dict["save_path"].resolve()).as_posix() dir_path = (url_dict["save_path"].resolve()).as_posix()
if platform == "win32": if platform == "win32":
dir_path = dir_path.replace("/", "\\") dir_path = dir_path.replace("/", "\\")
@@ -1373,53 +1332,6 @@ async def max_weight(lista):
# *=========================================== Define Commands # *=========================================== Define Commands
@client.hybrid_command(
name="skip_track",
description="Przeskocz kawałek w radiu",
guild=discord.Object(id=664789470779932693),
)
async def skip_track(ctx):
async with ctx.typing():
allowed = False
for role in ctx.author.roles:
if role.name in ("Thane", "Jarl"):
allowed = True
if not allowed:
await ctx.send("Łapy precz od radia")
else:
coroutine = asyncio.to_thread(
requests.get,
f"{RADIO_HARBOR_ADDRESS}{SKIP_TRACK}",
timeout=360,
)
result = await coroutine
logger.info("Done %s", result)
await ctx.send("Zrobione szefie!")
@client.hybrid_command(
name="zrestartuj_radio",
description="Komenda ktora uruchamia radio ponownie jakby się zawiesiło",
guild=discord.Object(id=664789470779932693),
)
async def zrestartuj_radio(ctx):
async with ctx.typing():
allowed = False
for role in ctx.author.roles:
if role.name in ("Thane", "Jarl"):
allowed = True
if not allowed:
await ctx.send("Łapy precz od radia")
else:
retcode = subprocess.run(
"/home/pi/Conjurer/restart_radio.sh",
shell=False,
check=False,
capture_output=True,
)
logger.info("Wynik: %s", retcode)
@client.hybrid_command( @client.hybrid_command(
name="dej_co_ze_spotifaja", name="dej_co_ze_spotifaja",
description="Podaj link do spotify - sciagnie i doda muzyke", description="Podaj link do spotify - sciagnie i doda muzyke",
@@ -1661,7 +1573,6 @@ async def graj_muzyko(ctx):
logger.info("Press play on tape completed") logger.info("Press play on tape completed")
check_music.start() check_music.start()
@client.hybrid_command( @client.hybrid_command(
name="radio_hardkor", name="radio_hardkor",
description="Włącza radio Conjurer na kanale #nocna-zmiana.", description="Włącza radio Conjurer na kanale #nocna-zmiana.",
@@ -1686,10 +1597,9 @@ async def radio_hardkor(ctx):
voice_client = client.voice_clients[0] voice_client = client.voice_clients[0]
voice_client.play( voice_client.play(
discord.PCMVolumeTransformer( discord.PCMVolumeTransformer(
discord.FFmpegPCMAudio("http://95.175.16.246:666/mp3-stream"), discord.FFmpegPCMAudio("http://95.175.16.246:666/mp3-stream"), volume=0.3
volume=0.3, )
) )
)
logger.info("Press play on radio completed") logger.info("Press play on radio completed")
else: else:
logger.error("Already playing") logger.error("Already playing")
@@ -1697,6 +1607,7 @@ async def radio_hardkor(ctx):
check_music.start() check_music.start()
@client.hybrid_command( @client.hybrid_command(
name="cisza", name="cisza",
description="Wyłącza muzykę i czyści plejliste.", description="Wyłącza muzykę i czyści plejliste.",
@@ -1839,7 +1750,6 @@ async def zagraj_mi_kawalek(ctx):
"Obawiam się że nie mogę nic znaleźć u siebie. Spróbuj komendy '$dej_co_ze_spotifaja' jeśli masz link" "Obawiam się że nie mogę nic znaleźć u siebie. Spróbuj komendy '$dej_co_ze_spotifaja' jeśli masz link"
) )
@client.hybrid_command( @client.hybrid_command(
name="dodaj_do_ulubionych", name="dodaj_do_ulubionych",
description="Dodaje do listy ulubionych w radiu", description="Dodaje do listy ulubionych w radiu",
@@ -1871,11 +1781,7 @@ async def dodaj_do_ulubionych(ctx):
json=jrequest, json=jrequest,
timeout=360, timeout=360,
) )
result = await coroutine _ = await coroutine
logger.info("Done %s", result)
await ctx.send("Zrobione szefie!")
@client.hybrid_command( @client.hybrid_command(
name="ja_chciol", name="ja_chciol",
description="Dodaje do listy ulubionych w radiu", description="Dodaje do listy ulubionych w radiu",
@@ -1905,47 +1811,7 @@ async def request_radio(ctx):
json=jrequest, json=jrequest,
timeout=360, timeout=360,
) )
result = await coroutine _ = await coroutine
logger.info("Done %s", result)
await ctx.send("Zrobione szefie!")
@client.hybrid_command(
name="stworz_audycje",
description="Dodaje do listy ulubionych w radiu",
guild=discord.Object(id=664789470779932693),
)
async def stworz_audycje(ctx):
"""
Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
Rest of the line are search terms.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. This parameter is required in all
Discord.py commands
"""
async with ctx.typing():
logger.info("Zaczynam szukać timestamp %s", datetime.now())
dlugosc_playlisty = ctx.message.content.split()[1]
word_list = ctx.message.content.split()
jrequest = {
"lista_slow": word_list,
"dlugosc_plejlisty": dlugosc_playlisty,
"UUID": str(uuid.uuid4()),
}
coroutine = asyncio.to_thread(
requests.post,
f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}",
json=jrequest,
timeout=360,
)
result = await coroutine
logger.info("Done %s", result)
await ctx.send("Zrobione szefie!")
@client.hybrid_command( @client.hybrid_command(
name="wyczysc_ulubione", name="wyczysc_ulubione",
description="Czysci liste ulubionych w radiu", description="Czysci liste ulubionych w radiu",
@@ -1963,13 +1829,12 @@ async def wyczysc_ulubione(ctx):
""" """
async with ctx.typing(): async with ctx.typing():
coroutine = asyncio.to_thread( coroutine = asyncio.to_thread(
requests.get, requests.post,
f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}", f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
json={},
timeout=360, timeout=360,
) )
result = await coroutine _ = await coroutine
logger.info("Done %s", result)
await ctx.send("Zrobione szefie!")
@client.hybrid_command( @client.hybrid_command(
@@ -2349,18 +2214,11 @@ async def wyszukaj_linki_do_dokumentow(ctx):
""" """
query = ctx.message.content query = ctx.message.content
query_uuid = uuid.uuid4() query_uuid = uuid.uuid4()
# TODO: TESTING ONLY!! #TODO: TESTING ONLY!!
# query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}') #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
ctx.message.content = ctx.message.content.replace( ctx.message.content = ctx.message.content.replace("$wyszukaj_linki_do_dokumentow", "")
"$wyszukaj_linki_do_dokumentow", ""
)
json_query = { json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": False}
"UUID": str(query_uuid),
"query": str(query),
"page": 1,
"deep_search": False,
}
coroutine = asyncio.to_thread( coroutine = asyncio.to_thread(
requests.post, requests.post,
f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
@@ -2368,16 +2226,14 @@ async def wyszukaj_linki_do_dokumentow(ctx):
timeout=360, timeout=360,
) )
await ctx.send( await ctx.send(
"*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" "*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
+ " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji." " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."+
+ " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować" " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować"
) )
query_response = await coroutine query_response = await coroutine
if not query_response.status_code == 200: if not query_response.status_code == 200:
await ctx.send( await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+
"*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić." " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało")
+ " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało"
)
return return
query, query_uuid, queue_size = ( query, query_uuid, queue_size = (
@@ -2391,11 +2247,8 @@ async def wyszukaj_linki_do_dokumentow(ctx):
username = ctx.message.author.name username = ctx.message.author.name
query_object = QueryControl(username, query_uuid, query, logger, ctx) query_object = QueryControl(username, query_uuid, query, logger, ctx)
OUT_COMM_Q.put(query_object) OUT_COMM_Q.put(query_object)
await ctx.send( await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+
f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce." " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.")
+ " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni."
)
@client.hybrid_command( @client.hybrid_command(
name="glebokie_gardlo", name="glebokie_gardlo",
@@ -2439,23 +2292,13 @@ async def wyszukaj_linki_do_dokumentow_deep(ctx):
result, MESSAGE_TABLE = await handle_response( result, MESSAGE_TABLE = await handle_response(
prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION" prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION"
) )
if len(result) < 1500: await ctx.send(result)
await ctx.send(result)
else:
while len(result) > 1500:
await ctx.send(result[:1500])
result = result[1500:]
return return
query_uuid = uuid.uuid4() query_uuid = uuid.uuid4()
# TODO: TESTING ONLY!! #TODO: TESTING ONLY!!
# query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}') #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
json_query = { json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": True}
"UUID": str(query_uuid),
"query": str(query),
"page": 1,
"deep_search": True,
}
coroutine = asyncio.to_thread( coroutine = asyncio.to_thread(
requests.post, requests.post,
f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
@@ -2463,16 +2306,14 @@ async def wyszukaj_linki_do_dokumentow_deep(ctx):
timeout=360, timeout=360,
) )
await ctx.send( await ctx.send(
"*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" "*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
+ " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie" " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"+
+ " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego" " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego"
) )
query_response = await coroutine query_response = await coroutine
if not query_response.status_code == 200: if not query_response.status_code == 200:
await ctx.send( await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+
"*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić." " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało")
+ " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało"
)
return return
query, query_uuid, queue_size = ( query, query_uuid, queue_size = (
@@ -2486,10 +2327,9 @@ async def wyszukaj_linki_do_dokumentow_deep(ctx):
username = ctx.message.author.name username = ctx.message.author.name
query_object = QueryControl(username, query_uuid, query, logger, ctx) query_object = QueryControl(username, query_uuid, query, logger, ctx)
OUT_COMM_Q.put(query_object) OUT_COMM_Q.put(query_object)
await ctx.send( await ctx.send(f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."+
f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze." " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*")
+ " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*"
)
@client.hybrid_command( @client.hybrid_command(
@@ -2521,29 +2361,11 @@ async def krecimy_pornola(ctx):
sciezka, files = await get_file(ctx, "Pornol", item_yt) sciezka, files = await get_file(ctx, "Pornol", item_yt)
if files: if files:
logger.info("Pornol udany") logger.info("Pornol udany")
await ctx.send(f"Jest w tajnym archiwum pod adresem{sciezka})") ctx.reply(f"Jest w tajnym archiwum pod adresem{sciezka})")
else: else:
await ctx.reply("Jak ładnie szefa poprosisz.") ctx.reply("Jak ładnie szefa poprosisz.")
@client.hybrid_command(
name="co_na_plejliscie_wariacie",
description="Wyswietl kawalki ktore sa na pocztku list radia",
guild=discord.Object(id=664789470779932693),
)
async def co_na_plejliscie_wariacie(ctx):
"""
Download a music number from youtube.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. It allows the command to interact
with the Discord API and
"""
await ctx.send("Dej mnie chwilkę")
async with ctx.typing():
logger.info("plejlista")
logger.info(PREPPED_TRACKS)
await ctx.send(f"Didżej Hammer i jego sztuczna inteligencja (||he he||) prezentują: \n- obecnie grany jest {(PREPPED_TRACKS['now_playing']).rstrip()} \n- na liście priorytetowej {(PREPPED_TRACKS['priority']).rstrip()} \n- na liście hitów obecnie czeka kawałek {(PREPPED_TRACKS['hit']).rstrip()} \n- na liście wszystkich kawałków {(PREPPED_TRACKS['all']).rstrip()} \n- najświeższym zamówieniem od słuchaczy jest {(PREPPED_TRACKS['requests']).rstrip()} \nZ kolei jingiel to: {(PREPPED_TRACKS['jingles']).rstrip()}")
# *================================== Run # *================================== Run
if __name__ == "__main__": if __name__ == "__main__":
@@ -2551,16 +2373,23 @@ if __name__ == "__main__":
threads = [] threads = []
logger.info("Starting discord bot: Creating threads") logger.info("Starting discord bot: Creating threads")
threads.append(threading.Thread(target=client.run, args=(TOKEN,))) threads.append(threading.Thread(target=client.run, args=(TOKEN,)))
threads.append(threading.Thread(target=comm_subroutine)) threads.append(threading.Thread(target=comm_subroutine, args=(logger,)))
logger.info("Starting discord bot: Starting threads") logger.info("Starting discord bot: Starting threads")
WRK_CNT = 0 WRK_CNT = 0
for worker in threads: for worker in threads:
WRK_CNT += 1 WRK_CNT+=1
logger.info("Starting discord bot: Starting thread %s", WRK_CNT) logger.info("Starting discord bot: Starting thread %s", WRK_CNT)
worker.start() worker.start()
logger.info("Starting discord bot: Joining threads") logger.info("Starting discord bot: Joining threads")
WRK_CNT = 0 WRK_CNT=0
for worker in threads: for worker in threads:
WRK_CNT += 1 WRK_CNT+=1
logger.info("Starting discord bot: Joining thread %s", WRK_CNT) logger.info("Starting discord bot: Joining thread %s", WRK_CNT)
worker.join() worker.join()
# widok - generuje się 5 stron po 4 linki
# odśwież - zamienić na "dalej" - zaciąga kolejne 20 rezultatów
# po wyslaniu czekamy na odbior i dodajemy UUID oraz nick poszukiwacza do queue ktora jest sprawdzana
# flask sobie chodzi i jak dostaje posta z wynikami to wyciaga z queue UUID i odpowiada że znalazła
# po samym wyszukaniu dajemy info że "Zaczynam szukać ale pare godzin to potrwa."
# OPCJONALNIE: Dosyłamy znalezione wyniki na bieżąco - wtedy wkładamy do queue dla UUID wyniki
+28 -58
View File
@@ -13,26 +13,16 @@ IN_COMM_Q = Queue()
awaiting_q = [] awaiting_q = []
incoming_q = Queue() incoming_q = Queue()
app = Flask(__name__) app = Flask(__name__)
PREPPED_TRACKS = {
"requests" : "",
"hit" : "",
"all": "",
"priority":"",
"jingles":"",
"now_playing":""
}
logger = logging.getLogger("discord")
class QueryControl: class QueryControl:
""" """
This class `QueryControl` is used to manage queries with information about the author, UUID, This class `QueryControl` is used to manage queries with information about the author, UUID,
content, logger, context, and replies. content, logger, context, and replies.
""" """
def __init__(self, query_author, query_uuid, query_content, ctx) -> None: def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
self.author = query_author self.author = query_author
self.uuid = query_uuid self.uuid = query_uuid
self.content = query_content self.content = query_content
self.logger = logging.getLogger("discord") self.logger = uplogger
self.stop = False self.stop = False
self.ctx = ctx self.ctx = ctx
self.logger.info( self.logger.info(
@@ -40,18 +30,6 @@ class QueryControl:
) )
self.replies = [] self.replies = []
@app.route("/prepped_tracks", methods=["POST"])
def log_radio_tracks():
logger = logging.getLogger("discord")
logger.info(request)
record = json.loads(request.data)
logger.info(record)
PREPPED_TRACKS[record[0]] = record[1]
logger.info("DATA RECEIVED")
return jsonify("SUCCESS")
@app.route("/conjurer", methods=["POST"]) @app.route("/conjurer", methods=["POST"])
def answer_external_command(): def answer_external_command():
""" """
@@ -60,11 +38,10 @@ def answer_external_command():
:return: The function `answer_external_command()` is returning a JSON response with the message :return: The function `answer_external_command()` is returning a JSON response with the message
"SUCCESS". "SUCCESS".
""" """
logger = logging.getLogger("discord") app.logger.info(request)
logger.info(request)
record = json.loads(request.data) record = json.loads(request.data)
logger.info(record) app.logger.info(record)
logger.info("DATA RECEIVED") app.logger.info("DATA RECEIVED")
incoming_q.put(record) incoming_q.put(record)
return jsonify("SUCCESS") return jsonify("SUCCESS")
@@ -79,29 +56,25 @@ def check_alive():
""" """
return jsonify("ALIVE") return jsonify("ALIVE")
def flask_debug(): def flask_debug(_logger):
""" """
The `flask_debug` function starts a Flask application in debug mode without using the reloader. The `flask_debug` function starts a Flask application in debug mode without using the reloader.
Do not use for production for fucks sake. Do not use for production for fucks sake.
""" """
logger = logging.getLogger("discord") _logger.info("Attempt debug")
logger.info("Attempt debug")
# trunk-ignore(bandit/B201) # trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS) app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run(): def waitress_run(_logger):
""" """
The `waitress_run` function serves the `app` on host "0.0.0.0" The `waitress_run` function serves the `app` on host "0.0.0.0"
and port 5000 using the Waitress WSGI server. and port 5000 using the Waitress WSGI server.
""" """
logger = logging.getLogger("discord") _logger.info("Attempt waitress")
logger.info("Attempt waitress")
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
def scan_queue(): def scan_queue(_logger):
""" """
The function `scan_queue` reads data from a queue, logs it, and appends it to another queue. The function `scan_queue` reads data from a queue, logs it, and appends it to another queue.
@@ -110,13 +83,12 @@ def scan_queue():
and provide information for debugging purposes. In this code snippet, the `_logger` object is used and provide information for debugging purposes. In this code snippet, the `_logger` object is used
to log the to log the
""" """
logger = logging.getLogger("discord")
while True: while True:
data = OUT_COMM_Q.get() data = OUT_COMM_Q.get()
logger.info(data) _logger.info(data)
awaiting_q.append(data) awaiting_q.append(data)
def scan_incoming(): def scan_incoming(_logger):
""" """
The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data
is found. is found.
@@ -126,29 +98,27 @@ def scan_incoming():
debugging, monitoring, or tracking the flow of the program. In this case, the `_logger` is being debugging, monitoring, or tracking the flow of the program. In this case, the `_logger` is being
used used
""" """
logger = logging.getLogger("discord")
while True: while True:
try: try:
answer = incoming_q.get(block=False) answer = incoming_q.get(block=False)
logger.info("DATA FOUND") _logger.info("DATA FOUND")
record_stored = False
for record in awaiting_q: for record in awaiting_q:
if record.uuid in answer.keys(): if record.uuid in answer.keys():
record_stored = True
record.stop = True
record.entries = answer[record.uuid]
IN_COMM_Q.put(record)
if not record_stored:
for key in answer.keys():
record = QueryControl("Orphaned", key, "Orphan", None)
record.stop = True record.stop = True
record.entries = answer[record.uuid] record.entries = answer[record.uuid]
IN_COMM_Q.put(record) IN_COMM_Q.put(record)
else:
for key in answer.keys():
record = QueryControl("Orphaned", key, "Orphan", _logger, None)
record.stop = True
record.entries = answer[record.uuid]
record.uuid = "000000"
IN_COMM_Q.put(record)
except Empty: except Empty:
time.sleep(1) time.sleep(1)
def comm_subroutine(): def comm_subroutine(logger):
""" """
The `comm_subroutine` function starts multiple threads to run different tasks concurrently. The `comm_subroutine` function starts multiple threads to run different tasks concurrently.
@@ -157,13 +127,12 @@ def comm_subroutine():
the provided code snippet, the logger is used to log messages at the "info" level the provided code snippet, the logger is used to log messages at the "info" level
""" """
#logger.setLevel(logging.DEBUG) #logger.setLevel(logging.DEBUG)
logger = logging.getLogger("discord") logger.info("Started")
logger.info("Started comms")
threads = [] threads = []
#threads.append(threading.Thread(target=flask_debug)) #threads.append(threading.Thread(target=flask_debug, args=(logger,)))
threads.append(threading.Thread(target=waitress_run)) threads.append(threading.Thread(target=waitress_run, args=(logger,)))
threads.append(threading.Thread(target=scan_queue)) threads.append(threading.Thread(target=scan_queue,args=(logger,)))
threads.append(threading.Thread(target=scan_incoming)) threads.append(threading.Thread(target=scan_incoming,args=(logger,)))
for worker in threads: for worker in threads:
@@ -172,4 +141,5 @@ def comm_subroutine():
worker.join() worker.join()
if __name__ == "__main__": if __name__ == "__main__":
comm_subroutine() logging_client = logging.getLogger(__name__)
comm_subroutine(logging_client)
+4 -9
View File
@@ -26,7 +26,6 @@ from queue import Queue
import requests import requests
import scrape_bot import scrape_bot
import search_bot import search_bot
#import search_bot2 as search_bot
from flask import Flask, jsonify, request from flask import Flask, jsonify, request
from habanero import Crossref from habanero import Crossref
from waitress import serve from waitress import serve
@@ -38,10 +37,7 @@ PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer" SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147" BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
MAX_CR_RESULTS = 7000
MAX_CR_RESULTS = 1000
#TEST PURPOSES ONLY!
#MAX_CR_RESULTS = 5
ENCODING = "utf-8" ENCODING = "utf-8"
@@ -118,15 +114,14 @@ class Librarian(object):
self.app.logger.info("STARTED SEARCH") self.app.logger.info("STARTED SEARCH")
if not deep_search: if not deep_search:
query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000 cr_result = self.cr.works(query=query, limit=1000)
cr_result = self.cr.works(query=query, limit=query_limit)
self.search_result_from_cr.update(cr_result) self.search_result_from_cr.update(cr_result)
self.total = cr_result["message"]["total-results"] self.total = cr_result["message"]["total-results"]
self.fetched += len(cr_result["message"]["items"]) self.fetched += len(cr_result["message"]["items"])
self.app.logger.info(self.total) self.app.logger.info(self.total)
self.app.logger.info(self.fetched) self.app.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched: while self.total > self.fetched and self.limit > self.fetched:
tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched) tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched)
cr_result["message"]["items"].extend(tmp_result["message"]["items"]) cr_result["message"]["items"].extend(tmp_result["message"]["items"])
self.total = tmp_result["message"]["total-results"] self.total = tmp_result["message"]["total-results"]
self.fetched = len(cr_result["message"]["items"]) self.fetched = len(cr_result["message"]["items"])
@@ -135,7 +130,7 @@ class Librarian(object):
time.sleep(0.1) time.sleep(0.1)
else: else:
cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True) cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True)
result = cr_result[0] result = cr_result[0]
for item in cr_result[1:]: for item in cr_result[1:]:
result["message"]["items"].extend(item["message"]["items"]) result["message"]["items"].extend(item["message"]["items"])
-8
View File
@@ -1,8 +0,0 @@
FILEPTH = r"C:\Database\chunks\4_chunk.txt"
with open(FILEPTH, "r", encoding="utf-8") as file:
tab = file.readlines()
print(len(tab))
#1842295
@@ -1,3 +1,3 @@
habanero habanero
waitress waitress
flask[async] flask
+6 -8
View File
@@ -33,14 +33,12 @@ def load_ndb_to_q(logger):
try: try:
ndb_database = json.load(ndb_file) ndb_database = json.load(ndb_file)
for _ in range (1,10): for _ in range (1,10):
try: key = next(iter(ndb_database))
key = next(iter(ndb_database)) item = ndb_database.pop(key)
_ = ndb_database.pop(key) logger.info(key)
logger.info(key) url = f"https://sci-hub.se/{key}"
url = f"https://sci-hub.se/{key}" WORK_Q.put([item, url, False])
WORK_Q.put([key, url, False])
except StopIteration:
break
ndb_file.truncate(0) ndb_file.truncate(0)
ndb_file.seek(0) ndb_file.seek(0)
json.dump(ndb_database, ndb_file, indent=4) json.dump(ndb_database, ndb_file, indent=4)
+25 -59
View File
@@ -21,9 +21,8 @@ Global Variables:
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry # TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
from queue import Empty, Queue from queue import Empty, Queue
from threading import Thread from threading import Thread
import time
q = Queue() q = Queue()
#TODO: Count number of lines in files and print to approximate on which part of the file search is
# DATA FOR TEST ONLY # DATA FOR TEST ONLY
# MAXTHREADS = 5 # MAXTHREADS = 5
@@ -36,7 +35,7 @@ DATABASE_PATH = r"C:\\Database\\chunks\\"
ENCODING = "utf-8" ENCODING = "utf-8"
CHUNK = "_chunk.txt" CHUNK = "_chunk.txt"
_sentinel = object() _sentinel = object()
WORK_Q_SIZE = 35500000 WORK_Q_SIZE = 15500000
def producer(out_q, control_q, filename, _logger): def producer(out_q, control_q, filename, _logger):
@@ -50,17 +49,9 @@ def producer(out_q, control_q, filename, _logger):
_logger: Logger object for logging. _logger: Logger object for logging.
""" """
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file: with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
print(f"Worker {filename} ") _logger.info(f"Worker {filename} ")
line_no = 0
while True: while True:
line = operated_file.readline() line = operated_file.readline()
line_no += 1
print(f"\t \t \t \t \t \t W{filename}{line_no}\r", end="")
if not line:
print(f"EOF {filename}")
break
if not line: if not line:
break break
out_q.put(line) out_q.put(line)
@@ -68,17 +59,16 @@ def producer(out_q, control_q, filename, _logger):
check = control_q.get(block=False) check = control_q.get(block=False)
except Empty: except Empty:
check = False check = False
if check is _sentinel: if check is _sentinel:
print("TERM signal received") _logger.info("TERM signal received")
control_q.put(check) control_q.put(check)
break break
print(f"Worker finished: {filename}") _logger.info(f"Worker finished: {filename}")
out_q.put(_sentinel) out_q.put(_sentinel)
# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK # IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger): def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _logger):
""" """
Consumes items from an input queue and checks if DOI exists in the live results list. Consumes items from an input queue and checks if DOI exists in the live results list.
@@ -89,49 +79,29 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
live_results (list): List to store the search results. live_results (list): List to store the search results.
_logger: Logger object for logging. _logger: Logger object for logging.
""" """
print(f"Consumer thread started: {no} no") for item in doi:
empty_counter = 0 result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
alive_no = 0
while True: while True:
done_check = True done_check = True
try: data = in_q.get()
data = in_q.get(block=True, timeout = 1) if data is _sentinel:
if data is _sentinel: _logger.info("Worker finished")
control_dict["sentinels"] += 1 control_dict["sentinels"] += 1
print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}") else:
for item in result_list:
else: if item["DOI"] in data and not item["exists"]:
empty_counter = 0 _logger.info(data)
alive_no += 1 _logger.info("HIT")
print(f"C{no}__{alive_no}\r", end="") item["exists"] = True
live_results.append(item)
for item in result_list: done_check = done_check and item["exists"]
if item["DOI"] in data and not item["exists"]: if done_check:
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}") control_q.put(_sentinel)
_logger.info(data)
_logger.info("HIT")
item["exists"] = True
live_results.append(item)
done_check = done_check and item["exists"]
if done_check:
control_q.put(_sentinel)
except Empty:
empty_counter += 1
time.sleep(1)
print(f"Consumer {no} empty")
if empty_counter > 5:
print("Consumer %s empty lvl 2", no)
time.sleep(2)
elif empty_counter > 10:
print(f"Consumer thread finished {no}")
break
if control_dict["sentinels"] >= MAXTHREADS: if control_dict["sentinels"] >= MAXTHREADS:
_logger.info(f"All workers finished {no}") _logger.info("All workers finished")
break break
def search_for_doi(doi, live_results, _logger): def search_for_doi(doi, live_results, _logger):
""" """
Search for DOI in live_results using _logger for logging. Search for DOI in live_results using _logger for logging.
@@ -146,13 +116,9 @@ def search_for_doi(doi, live_results, _logger):
threads = [] threads = []
work_q = Queue(maxsize=WORK_Q_SIZE) work_q = Queue(maxsize=WORK_Q_SIZE)
control_q = Queue() control_q = Queue()
for _ in range (1, len(doi)//1000+1):
for item in doi:
result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
for i in range (0, (len(doi)//1000)+2):
t_cons = Thread( t_cons = Thread(
target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger) target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, _logger)
) )
_logger.info("Consumer thread created") _logger.info("Consumer thread created")
threads.append(t_cons) threads.append(t_cons)
-206
View File
@@ -1,206 +0,0 @@
"""
This module contains functions for searching for DOI (Digital Object Identifier)
in a list of live results.
It includes a producer-consumer pattern implementation using threads and queues.
Functions:
- producer: Reads lines from a file and puts them into an output queue.
- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
Global Variables:
- MAXTHREADS: Maximum number of worker threads.
- DATABASE_PATH: Path to the database files.
- ENCODING: Encoding of the database files.
- CHUNK: File name pattern for the database files.
- _sentinel: Sentinel object used to signal termination.
- result_list: List to store the search results.
- WORK_Q_SIZE: Maximum size of the work queue.
"""
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
from multiprocessing import Manager, Process, Queue
from queue import Empty, Full
import time
# DATA FOR TEST ONLY
#MAXTHREADS = 6
#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
# DEPLOYMENT DATA
MAXTHREADS = 41
DATABASE_PATH = r"C:\\Database\\chunks\\"
ENCODING = "utf-8"
CHUNK = "_chunk.txt"
_sentinel = None
WORK_Q_SIZE = 355000
def producer(out_q, control_q, filename, _logger):
"""
Produces items from the output queue and puts them into the control queue.
Args:
out_q (Queue): Output queue.
control_q (Queue): Control queue.
filename (str): Name of the file.
_logger: Logger object for logging.
"""
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
print(f"Worker {filename} ")
line_no = 0
while True:
line = operated_file.readline()
line_no += 1
print(f"\t \t \t W{filename}{line_no}\r", end="")
if not line:
print(f"EOF {filename}")
break
try:
out_q.put([line, line_no, filename], block = True, timeout = 60)
except Full:
print("queue fulll")
try:
check = control_q.get(block=False)
except Empty:
check = False
if check is None:
print(f"TERM signal received {filename}")
control_q.put(check)
control_q.close()
break
print(f"Worker finished: {filename}")
out_q.put(_sentinel)
out_q.close()
# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
"""
Consumes items from an input queue and checks if DOI exists in the live results list.
Args:
in_q (Queue): Input queue.
control_q (Queue): Control queue.
doi (list): List of DOI to search for.
live_results (list): List to store the search results.
_logger: Logger object for logging.
"""
print(f"Consumer thread started: {no} no")
empty_counter = 0
alive_no = 0
while True:
done_check = True
try:
data = in_q.get(block=True, timeout = 1)
empty_counter = 0
if data is None:
control_dict["sentinels"] += 1
print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
else:
empty_counter = 0
if alive_no > 9:
alive_no = 0
alive_no += 1
print(f"C{no}{alive_no}\r", end="")
for item in result_list:
if item["DOI"] in data[0] and not item["exists"]:
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
item["exists"] = True
live_results.append(item)
done_check = done_check and item["exists"]
if done_check:
control_q.put(_sentinel)
except Empty:
empty_counter += 1
time.sleep(1)
print(f"Consumer {no} empty")
if empty_counter > 5:
print("Consumer %s empty lvl 2", no)
time.sleep(2)
elif empty_counter > 10:
print(f"Consumer thread finished {no}")
break
if control_dict["sentinels"] >= MAXTHREADS:
print(f"All workers finished killing {no}")
break
print("Consumer thread finished %s",no )
control_q.close()
def search_for_doi(doi, live_results, _logger):
"""
Search for DOI in live_results using _logger for logging.
Args:
doi (list): List of DOI to search for.
live_results (list): List to store the search results.
_logger: Logger object for logging.
"""
with Manager() as manager:
control_dict = manager.dict()
control_dict["sentinels"] = 0
result_list = []
for item in doi:
result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
result_list_proxy = manager.list(result_list)
threads = []
work_q = Queue(maxsize=WORK_Q_SIZE)
control_q = Queue()
for i in range(0, MAXTHREADS):
# TEST DATA
#filename = "test" + str(i) + CHUNK
# DEPLOYMENT DATA
filename = str(i) + CHUNK
print(f"Creating worker thread no: {i}")
threads.append(
Process(target=producer, args=(work_q, control_q, filename, _logger))
)
for i in range (0, (len(doi)//1000)+5):
t_cons = Process(
target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger)
)
print("Consumer thread created")
threads.append(t_cons)
for worker in threads:
worker.start()
for worker in threads:
worker.join()
result_list = list(result_list_proxy)
for worker in threads:
worker.close()
return result_list
if __name__ == "__main__":
import logging
logger = logging.getLogger()
logger.setLevel("DEBUG")
h1 = logging.StreamHandler()
logger.addHandler(h1)
print("TEST RUN")
live_result = []
print(
search_for_doi(
[
("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
("10.2165/00128415-200309690-00017", "DATA"),
("10.5772/48313", "DATA"),
("10.15803/ijnc.7.2_419", "DATA"),
("10.1051/0004-6361/201321596e", "DATA"),
("10.2307/40835941", "DATA"),
("NOT_ON_LIST", "DATA"),
],
live_result,
logger,
)
)
print(live_result)
-45
View File
@@ -1,45 +0,0 @@
import asyncio
import json
import logging
from json.decoder import JSONDecodeError
from logging import handlers
import requests
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
async def send_results():
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
h1 = handlers.RotatingFileHandler(
filename="D:\\logs\\librarian.log",
encoding="utf-8",
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
)
logger.addHandler(h1)
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
except JSONDecodeError:
pass
result = database["4bacbca0-ce3b-45c6-90a1-05c70b46b740"]
result = {"4bacbca0-ce3b-45c6-90a1-05c70b46b740":result}
coroutine = asyncio.to_thread(
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
logger.info("SENT")
result = await coroutine
logger.info(result.status_code)
logger.info("SEND CONFIRMED")
async def main():
await send_results()
if __name__ == "__main__":
asyncio.run(main())
+58 -143
View File
@@ -6,9 +6,7 @@ API endpoints.
""" """
import json import json
import logging import logging
import os
import re import re
import requests
import threading import threading
import time import time
from datetime import datetime from datetime import datetime
@@ -16,11 +14,10 @@ from logging import handlers
from pathlib import Path from pathlib import Path
from platform import uname from platform import uname
from sys import platform from sys import platform
from flask import Flask, jsonify, redirect, request, send_from_directory, render_template
from flask import Flask, jsonify, redirect, request, send_from_directory
from waitress import serve from waitress import serve
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
MUSIC_TRACKER = "/prepped_tracks"
HOST_ADDRESS = "192.168.1.15" HOST_ADDRESS = "192.168.1.15"
PORT_ADDRESS = 5000 PORT_ADDRESS = 5000
if platform in ("linux", "linux2"): if platform in ("linux", "linux2"):
@@ -31,8 +28,6 @@ if platform in ("linux", "linux2"):
NETRC_FILE = "/home/mtuszowski/.netrc" NETRC_FILE = "/home/mtuszowski/.netrc"
LOGSTORE = "/home/mtuszowski/conjurer/logs/" LOGSTORE = "/home/mtuszowski/conjurer/logs/"
ENCODING = "utf-8" ENCODING = "utf-8"
RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
else: else:
LOGFILE = "/home/pi/Conjurer/discord_mus_service.log" LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
@@ -41,20 +36,52 @@ if platform in ("linux", "linux2"):
ENCODING = "utf-8" ENCODING = "utf-8"
MUSIC_FOLDER = "/home/pi/RetroPie/mp3/" MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/" PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/"
RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
music_file_list = [] music_file_list = []
priority_list = [] priority_list = []
def create_playlist():
"""
Reads a JSON file containing a playlist and writes the playlist items to a text file.
The JSON file path is '/home/pi/Conjurer/playlist.json',
and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
Raises:
JSONDecodeError: If the JSON file cannot be decoded.
"""
with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
with open(
"/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
) as w_file:
try:
playlist = json.load(r_file)
for item in playlist:
w_file.write(item)
w_file.write("\n")
except json.JSONDecodeError:
pass
with open("/home/pi/Conjurer/prio_playlist.json", "r+", encoding="utf-8") as r_file:
with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
try:
playlist = json.load(r_file)
for item in playlist:
w_file.write(item)
w_file.write("\n")
except json.JSONDecodeError:
pass
def rescan(): def rescan():
""" """
The `rescan` function logs a message, scans for mp3 files in a specified folder, The `rescan` function logs a message, scans for mp3 files in a specified folder,
and adds them to a list of music files. and adds them to a list of music files.
""" """
logger = logging.getLogger("conjurer_musician") logging.info("Rescan triggered")
logger.info("Rescan triggered")
for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"): for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
temp_music_file = mp3_item.as_posix() temp_music_file = mp3_item.as_posix()
@@ -68,22 +95,18 @@ def rescan():
temp_music_file = temp_music_file.replace("/", "\\") temp_music_file = temp_music_file.replace("/", "\\")
priority_list.append(temp_music_file) priority_list.append(temp_music_file)
with open( with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
"/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8" database = music_file_list
) as w_file: # self.app.logger.info("DUMPING DATA")
try: s_file.truncate(0)
for item in music_file_list: s_file.seek(0)
w_file.write(item) json.dump(database, s_file)
w_file.write("\n") with open("/home/pi/Conjurer/prio_playlist.json", "r+", encoding="utf-8") as s_file:
except json.JSONDecodeError: # self.app.logger.info("DUMPING DATA")
pass s_file.truncate(0)
with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file: s_file.seek(0)
try: json.dump(priority_list, s_file)
for item in priority_list: create_playlist()
w_file.write(item)
w_file.write("\n")
except json.JSONDecodeError:
pass
def thread_rescan(): def thread_rescan():
@@ -91,87 +114,13 @@ def thread_rescan():
The `thread_rescan` function periodically triggers a rescan operation after a specified time The `thread_rescan` function periodically triggers a rescan operation after a specified time
interval. interval.
""" """
logger = logging.getLogger("conjurer_musician") logging.info("Starting filesystemupdater")
logger.info("Starting filesystemupdater")
while True: while True:
time.sleep(60 * 60 * 24) time.sleep(60 * 60 * 24)
logger.info("Rescan triggered") logging.info("Rescan triggered")
rescan() rescan()
def scan_tracks():
#Set the filename and open the file
logger = logging.getLogger("conjurer_musician")
file = open(RADIOLOG_PATH,'r')
#Find the size of the file and move to the end
st_results = os.stat(RADIOLOG_PATH)
st_size = st_results[6]
file.seek(st_size)
st_results1 = os.stat(PERSISTENCE_PATH)
prev_st_size1 = st_results[6]
while 1:
st_results1 = os.stat(PERSISTENCE_PATH)
st_size1 = st_results1[6]
if prev_st_size1 != st_size1:
while prev_st_size1 != st_size1:
prev_st_size1 = st_size1
st_results1 = os.stat(PERSISTENCE_PATH)
st_size1 = st_results1[6]
time.sleep(0.1)
file1 = open(PERSISTENCE_PATH, 'r')
lines = file1.readlines()
result = ["now_playing", lines[2]]
file1.close()
returned = requests.post(
f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
json=result,
timeout=360)
logger.info("SENT")
logger.info(returned.status_code)
logger.info("SEND CONFIRMED")
where = file.tell()
line = file.readline()
if not line:
time.sleep(1)
file.seek(where)
else:
if re.match(".*Prepared.*",line):
result = None
if re.match(".*jingles.*", line):
logger.info("jingles")
logger.info(line) # already has newline
result = ["jingles", line]
elif re.match(".*priority.*", line):
logger.info("priority")
logger.info(line) # already has newline
result = ["priority", line]
elif re.match(".*hit.*", line):
logger.info("hit")
logger.info(line) # already has newline
result = ["hit", line]
elif re.match(".*all_playlist.*", line):
logger.info("all")
logger.info(line) # already has newline
result = ["all", line]
elif re.match(".*request.*", line):
logger.info("requests")
logger.info(line) # already has newline
result = ["requests", line]
if result:
returned = requests.post(
f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
json=result,
timeout=360)
logger.info("SENT")
logger.info(returned.status_code)
logger.info("SEND CONFIRMED")
time.sleep(0.1)
app = Flask(__name__) app = Flask(__name__)
@@ -289,7 +238,6 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
fun_logger.info("Wiele plików do zagrania") fun_logger.info("Wiele plików do zagrania")
search_weight.sort(key=lambda x: x[0], reverse=True) search_weight.sort(key=lambda x: x[0], reverse=True)
return_list.extend(search_weight[: int(how_many)]) return_list.extend(search_weight[: int(how_many)])
fun_logger.info("Done: %s", return_list)
return return_list return return_list
@@ -329,9 +277,8 @@ def stream_music():
:return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`. :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
""" """
return send_from_directory("/tmp/hls", "stream.m3u8")
#return send_from_directory("/tmp/hls", "stream.m3u8")
return render_template("/home/pi/Conjurer/stream.html")
@app.route("/<string:file_name>") @app.route("/<string:file_name>")
def stream(file_name): def stream(file_name):
@@ -344,10 +291,10 @@ def stream(file_name):
Returns: Returns:
Response: The response containing the streamed file. Response: The response containing the streamed file.
""" """
# trunk-ignore(bandit/B108)
video_dir = "/tmp/hls" video_dir = "/tmp/hls"
return send_from_directory(video_dir, file_name) return send_from_directory(video_dir, file_name)
@app.route("/stream_mp3", methods=["GET"]) @app.route("/stream_mp3", methods=["GET"])
def stream_music_mp3(): def stream_music_mp3():
""" """
@@ -366,10 +313,10 @@ def clear_pr_pls():
:return: A JSON response indicating the success of the operation. :return: A JSON response indicating the success of the operation.
""" """
app.logger.info("CLEARING PLAYLIST") with open(
with open('/home/pi/Conjurer/priority_queue.playlist', 'w', encoding='utf-8') as cleared_pl: "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
cleared_pl.write('') ) as s_file:
s_file.truncate(0)
return_data = jsonify(isError=False, message="Success", statusCode=200, data=[]) return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
return return_data, 200 return return_data, 200
@@ -453,37 +400,6 @@ def add_request():
) )
return return_data return return_data
@app.route("/create_priority_playlist", methods=["POST"])
def create_priority_playlist():
"""
The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received
item, adds it to a music file list, and returns a success message along with the updated record.
:return: A tuple containing a JSON response and a status code.
The JSON response includes keys `isError`,
`message`, `statusCode`, and `data`, with values
indicating the success of the operation and the
data that was received and added to the `music_file_list`.
The status code returned is 200,indicating a successful response.
"""
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["lista_slow"])
app.logger.info(record["UUID"])
app.logger.info(record["dlugosc_plejlisty"])
return_data = wyszukaj(
record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
)
with open(
"/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
) as s_file:
for item in return_data:
s_file.write(item[1] + "\n")
return_data = (
jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
200,
)
return return_data
@app.route("/add_to_priority", methods=["POST"]) @app.route("/add_to_priority", methods=["POST"])
@@ -568,7 +484,6 @@ if __name__ == "__main__":
# threads.append(threading.Thread(target=flask_debug)) # threads.append(threading.Thread(target=flask_debug))
threads.append(threading.Thread(target=waitress_run)) threads.append(threading.Thread(target=waitress_run))
threads.append(threading.Thread(target=thread_rescan)) threads.append(threading.Thread(target=thread_rescan))
threads.append(threading.Thread(target=scan_tracks))
for worker in threads: for worker in threads:
worker.start() worker.start()
-26
View File
@@ -1,26 +0,0 @@
# This Python file uses the following encoding: utf-8
"""This module contains utility scripts used by musi radiostation Conjurer"""
import time
import re
NAME = "/home/pi/Conjurer/persistence.log"
def print_top():
"""
Prints the top 65 lines from a file specified by the NAME variable.
"""
while True:
line = ""
buffer = ""
with open(NAME, "r", encoding="utf-8") as f:
for _ in range (45):
line = f.readline()
if re.match(".*mp3", line):
buffer += line
print("=====================================")
print(buffer)
print("=====================================")
time.sleep(180)
if __name__ == "__main__":
print_top()
-14
View File
@@ -1,14 +0,0 @@
$dodaj_do_ulubionych 100 nightwish
$dodaj_do_ulubionych 100 amon amarth
$dodaj_do_ulubionych 100 tactical sekt
$dodaj_do_ulubionych 100 lacuna coil
$dodaj_do_ulubionych 100 sefa
$dodaj_do_ulubionych 100 frenchcore
$dodaj_do_ulubionych 100 peacock
$dodaj_do_ulubionych 100 vavamuffin
$dodaj_do_ulubionych 100 abradab
$dodaj_do_ulubionych 100 eluveitie
$dodaj_do_ulubionych 100 youtube
$dodaj_do_ulubionych 100 elyose
$dodaj_do_ulubionych 1 artisans du chaos
+115 -143
View File
@@ -5,163 +5,135 @@
# Load icecast credentials from a JSON file # Load icecast credentials from a JSON file
let json.parse credentials = file.contents("/home/pi/Conjurer/icecast_credentials.json") let json.parse credentials = file.contents("/home/pi/Conjurer/icecast_credentials.json")
# Enable replaygain metadata processing # Enable replaygain metadata processing
enable_replaygain_metadata() enable_replaygain_metadata()
l = playlog(duration = 14400.0, persistency="/home/pi/Conjurer/persistence.log")
# Set up a playlog for tracking played tracks def check(r)
m = request.metadata(r)
l = playlog(duration = 72000.0, persistency="/home/pi/Conjurer/persistence.log") if l.last(m) < 3600. then
log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).")
# Function to check if a track can be played based on its metadata false
def check(r) else
m = request.metadata(r) l.add(m)
if l.last(m) < 36000. then true
log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).") end
false
else
l.add(m)
true
end end
end
# Define playlists to be used in the stream
s1 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/all_playlist.playlist"))
s2 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/priority_queue.playlist"))
s3 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/hit.playlist"))
# Create a request queue for user-generated requests
requests_queue = request.queue()
# Function to process the request queue and add new requests # Define playlists to be used in the stream
def queue_processing() s1 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/all_playlist.playlist"))
text=file.lines("/home/pi/Conjurer/request.playlist") s2 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/priority_queue.playlist"))
if text != [] then s3 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/hit.playlist"))
list.iter(fun(item) -> requests_queue.push.uri(item), text) requests_queue = request.queue()
file.remove("/home/pi/Conjurer/request.playlist") def queue_processing()
f = file.open("/home/pi/Conjurer/request.playlist", create=true) f1=file.read("/home/pi/Conjurer/request.playlist")
f.close() text = f1()
if text != "" then
requests_queue.push.uri(text)
file.remove("/home/pi/Conjurer/request.playlist")
f = file.open("/home/pi/Conjurer/request.playlist", create=true)
f.close()
log.info("done")
else
log.info("no requests")
end
end end
end thread.run(every=60., queue_processing)
# Randomly select a playlist with weights # Randomly select a playlist with weights
s4 = random(id="randomizer", weights=[2, 3, 5], [s1, s2, s3]) s = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3])
#s = delay(duration=5., s)
#radio = add([delay(1800., jingles), music])
#radio = fallback([delay(1800., jingles), music])
#p = predicate.signal()
#radio = switch([(p, jingles), ({true}, music)])
#thread.run(every=1200., {p.signal()})
# def cmd(_)
# p.signal()
# "Jingle inserted!"
# end
# server.register("insert_jingle", cmd)
# Load jingles playlist # Set up an interactive harbor for controlling the stream
jingles = (playlist(reload_mode="watch", "/home/pi/Conjurer/jingles.playlist")) interactive.harbor(port = 9999)
# Set up interactive controls for bass boost
f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.)
g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.)
b = bass_boost(frequency=f, gain=g, s)
s = add([s, b])
# Set up interactive control for main volume
a = interactive.float("main_volume", min=0., max=20., 1.)
s = compress.multiband.interactive(bands=3, s)
# Create the main stream with random playlist and jingles # Apply audio processing effects
s = rotate(id="randomizer", weights=[10, 1], [s4, jingles]) s = nrj(normalize(s))
s = crossfade(fade_out=3., fade_in=3., duration=5., s)
s = amplify(a, s)
s = crossfade(fade_out=2., fade_in=2., duration=4., fallback(id="switcher", track_sensitive=false, [requests_queue, s])) # Skip blank sections in the stream
s = blank.skip(max_blank=2., s)
# Set up an interactive harbor for controlling the stream # Configure logging settings
interactive.harbor(port = 9999) log_to_stdout = true
log_to_file = true
logpath = "/home/pi/Conjurer/radio_log.log"
loglevel = 4
set("log.stdout", log_to_stdout)
set("log.level", loglevel)
# Set up interactive controls for bass boost # Enable logging to file
f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.) set("log.file", log_to_file)
g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.) set("log.file.path", logpath)
b = bass_boost(frequency=f, gain=g, s)
s = add([s, b])
# Set up interactive control for main volume # Set up emergency fallback track
a = interactive.float("main_volume", min=0., max=20., 1.) emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
s = compress.multiband.interactive(bands=7, s)
# Apply audio processing effects # Create a fallback stream with the main stream and emergency track
s = nrj(normalize(s)) radio = fallback(id="switcher", track_sensitive=true, [requests_queue, s, emergency])
s = amplify(a, s)
mic = buffer(input.pulseaudio()) # Microphone # Handle metadata events
mic = gate(threshold=-30., range=-80., mic) handle_metadata = fun (m) -> begin
mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic) log.info(m["title"])
log.info(m["artist"])
s = add([s, mic]) log.info(m["filename"])
#w = interactive.float("wetness", min=0., max=1., 1.)
#s = dry_wet(w,s,s2)
# Skip blank sections in the stream
s = blank.skip(max_blank=10., s)
# Configure logging settings
log_to_stdout = true
log_to_file = true
logpath = "/home/pi/Conjurer/radio_log.log"
loglevel = 3
set("log.stdout", log_to_stdout)
set("log.level", loglevel)
# Enable logging to file
set("log.file", log_to_file)
set("log.file.path", logpath)
# Set up emergency fallback track
emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
radio = fallback(id="switcher2", track_sensitive=false, [s, emergency])
# Set up an interactive control for skipping tracks
p = interactive.bool("Skip track", false)
def http_skip(~protocol, ~data, ~headers, uri)=
radio.skip()
http.response(code=200,data="Skipped")
end
harbor.http.register(port=54321,method="GET","/skip", http_skip)
# Function to process the request queue and skip the current track if requested
def check_skip()
if p() then
log.info("Skipping current track.")
radio.skip()
p.set(false)
end end
end radio.on_metadata(handle_metadata)
# Run the queue processing function every 60 seconds # Enable persistent script parameters
thread.run(every=60., queue_processing) interactive.persistent("script.params")
# Configure output formats and destinations
# Run the check_skip function every 5 seconds output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
thread.run(every=15., check_skip) output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
# output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio)
# output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio)
# Enable persistent script parameters # output.icecast(%aac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="aac-stream", radio)
interactive.persistent("script.params") # output.icecast(%flac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="flac-stream", radio)
# output.icecast(%vorbis, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="vorbis-stream", radio)
# Configure output formats and destinations # output.icecast(%speex, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="speex-stream", radio)
# output.icecast(%wav, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="wav-stream", radio)
output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio) # output.icecast(%pcm, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="pcm-stream", radio)
output.pulseaudio(radio) # output.icecast(%raw, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="raw-stream", radio)
#output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio) # output.icecast(%s16l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16l-stream", radio)
# Uncomment the following lines to enable additional output formats # output.icecast(%s16b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16b-stream", radio)
# output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio) # output.icecast(%s24l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24l-stream", radio)
# output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio) # output.icecast(%s24b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24b-stream", radio)
# output.icecast(%aac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="aac-stream", radio) # output.icecast(%s32l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32l-stream", radio)
# output.icecast(%flac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="flac-stream", radio) # output.icecast(%s32b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32b-stream", radio)
# output.icecast(%vorbis, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="vorbis-stream", radio) # output.icecast(%s64l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64l-stream", radio)
# output.icecast(%speex, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="speex-stream", radio) # output.icecast(%s64b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64b-stream", radio)
# output.icecast(%wav, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="wav-stream", radio) # output.icecast(%s128l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128l-stream", radio)
# output.icecast(%pcm, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="pcm-stream", radio) # output.icecast(%s128b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128b-stream", radio)
# output.icecast(%raw, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="raw-stream", radio) # output.icecast(%s256l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256l-stream", radio)
# output.icecast(%s16l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16l-stream", radio) # output.icecast(%s256b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256b-stream", radio)
# output.icecast(%s16b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16b-stream", radio) # output.icecast(%s512l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512l-stream", radio)
# output.icecast(%s24l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24l-stream", radio) # output.icecast(%s512b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512b-stream", radio)
# output.icecast(%s24b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24b-stream", radio) # output.icecast(%s1024l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024l-stream", radio)
# output.icecast(%s32l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32l-stream", radio) # output.icecast(%s1024b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024b-stream", radio)
# output.icecast(%s32b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32b-stream", radio) # output.icecast(%s2048l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048l-stream", radio)
# output.icecast(%s64l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64l-stream", radio) # output.icecast(%s2048b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048b-stream", radio)
# output.icecast(%s64b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64b-stream", radio) # output.icecast(%s4096l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096l-stream", radio)
# output.icecast(%s128l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128l-stream", radio) # output.icecast(%s4096b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096b-stream", radio)
# output.icecast(%s128b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128b-stream", radio) # output.icecast(%s8192l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192l-stream", radio)
# output.icecast(%s256l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256l-stream", radio) # output.icecast(%s8192b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192b-stream", radio)
# output.icecast(%s256b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256b-stream", radio)
# output.icecast(%s512l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512l-stream", radio)
# output.icecast(%s512b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512b-stream", radio)
# output.icecast(%s1024l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024l-stream", radio)
# output.icecast(%s1024b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024b-stream", radio)
# output.icecast(%s2048l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048l-stream", radio)
# output.icecast(%s2048b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048b-stream", radio)
# output.icecast(%s4096l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096l-stream", radio)
# output.icecast(%s4096b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096b-stream", radio)
# output.icecast(%s8192l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192l-stream", radio)
# output.icecast(%s8192b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192b-stream", radio)
-9
View File
@@ -1,9 +0,0 @@
<html>
<head>
<title>HTTP Live Streaming Example</title>
</head>
<body>
<video src="/tmp/hls/stream.m3u8" height="300" width="400">
</video>
</body>
</html>
+1 -2
View File
@@ -12,5 +12,4 @@ spotipy
tiktoken tiktoken
PyNaCl PyNaCl
flask flask
waitress waitress
clickupython
-3
View File
@@ -1,3 +0,0 @@
sshpass -p raspberry ssh pi@192.168.1.15 << 'ENDSSH'
sudo reboot
ENDSSH
-7
View File
@@ -204,13 +204,6 @@
false, false,
false false
], ],
"same plusy\\b": [
"Jak na cmentarzu Szefie. Jak na cmentarzu",
15.0,
0,
false,
false
],
"kld\\b": [ "kld\\b": [
"KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.", "KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.",
15.0, 15.0,
-2
View File
@@ -239,8 +239,6 @@ def get_item_name(sp, item_type, item_id):
name = sp.album(album_id=item_id).get("name") name = sp.album(album_id=item_id).get("name")
elif item_type == "track": elif item_type == "track":
name = sp.track(track_id=item_id).get("name") name = sp.track(track_id=item_id).get("name")
elif item_type == "artist_top_tracks":
name = sp.artist_top_tracks(artist_id=item_id).get("name")
return sanitize(name) return sanitize(name)
-33
View File
@@ -1,33 +0,0 @@
import json
MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
ENCODING = "utf-8"
messages = [
{"role": "user", "content" : "Hammer: All systems operational"},
{"role": "assistant", "content": "Aye, Szefie! Wszystkie systemy działają sprawnie. Gotów do dalszej służby! Co podać?"},
{"role": "user", "content" : "Hammer: Podaj coś na orzeźwienie i przebudzenie. COś co rozgrzeje moje stare kości po ciężkiej pracy!"},
{"role": "assistant", "content": "Z przyjemnością Szefie! Dla Ciebie przygotuję Midzynarodówkę Interstelarną Wolf's Dragoon - wysoce energetyczny koktajl rozgrzewający i pobudzający jednocześnie."+
"Do przygotowania potrzebuję:"+
"\n"+
"60 ml mocnego rumu z Wolfa (lub innego, jeśli wolisz)"+
"30 ml soku z cytryny"+
"35 ml domowego syropu z mięty"+
"15 ml świeżo zmielonej kawy"+
"\n"+
"Mieszam wszystko w shakerze z lodem, a potem przelewam przez sitko do kieliszka koktajlowego. W górę serwuję z bigoskiem z miętą i zestem z cytryny. Rozgrzewa, orzeźwia i daje kopa na resztę nocy."+
"\n"+
"Co powiesz Szefie, podać Ci taki?"},
{"role": "user", "content" : "Hammer: Brzmi jak coś co zrobi robotę! Skal!"},
]
for temp in messages:
with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
# First we load existing data into a dict.
file_data = json.load(file_memory)
# Join new_data with file_data inside emp_details
file_data.append(temp)
file_memory.seek(0)
# convert back to json.
json.dump(file_data, file_memory, indent=4)