Refactoring step one

This commit is contained in:
2024-11-07 09:54:52 +01:00
parent a8e32d6b05
commit c4e0dea7da
9 changed files with 2782 additions and 2 deletions
View File
+26
View File
@@ -0,0 +1,26 @@
import tiktoken
def num_tokens_from_string(message, model):
"""
The function takes a string message and a model as input and returns the number of tokens in the
message according to the given model.
:param message: A string containing the message or text from which you want to count the number of
tokens
:param model: The model parameter refers to a language model or tokenizer that can be used to
tokenize the input string. It could be a pre-trained model or a custom tokenizer
"""
tokens_per_message = 3
tokens_per_name = 1
chat_gpt_encoding = tiktoken.encoding_for_model(model)
num_tokens = 0
num_tokens += tokens_per_message
for keys, values in message.items():
num_tokens += len(chat_gpt_encoding.encode(values))
if keys == "role":
num_tokens += tokens_per_name
num_tokens += 3 # every reply is primed with <|start|>assistant<|message|>
return num_tokens
+101
View File
@@ -0,0 +1,101 @@
import datetime
import json
from sys import platform
from typing import List, Optional, TypedDict
from platform import uname
Music_Config = TypedDict(
"Music_Config",
{
"ctx": Optional[str],
"queue": List[str],
"requester": List[str],
},
)
# *=========================================== Predefines
MASTER_TIMEOUT = datetime.now()
INITIAL_TIME_WAIT = 500
MUZYKA: Music_Config = {"ctx": None, "queue": [], "requester": []}
LOGFILE = ""
NETRC_FILE = ""
MUSIC_FOLDER = ""
MEMORY_FIVE_SIARA = ""
MEMORY_FIVE_MUZYKA = ""
SETTINGS_FILE = ""
ENCODING = ""
GRAPHICS_PATH = ""
MUZYKA_MOJEGO_LUDU_HISTORIA = 1500
MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15
MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321"
SKIP_TRACK = "/skip"
GET_MP3 = "/mp3"
SEND_MP3 = "/update_mp3"
GET_PLAYLIST = "/get_music"
ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
REQUEST_MUSIC = "/request_radio_file"
CLEAR_PRIO = "/clear_pr_pls"
LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
SEND_QUERY = "/query"
TIME_BETWEEN_CALLS = 100000
LAST_SPONTANEOUS_CALL = datetime.now()
HOST_ADDRESS = "192.168.1.191"
PORT_ADDRESS = 5000
# *=========================================== Platform Specific Predefines
if platform in ("linux", "linux2"):
SEPARATOR_FILE_PATH = "/"
if "microsoft-standard" in uname().release:
LOGFILE = "/home/mtuszowski/conjurer/discord.log"
MEMORY_FIVE_SIARA = "/home/mtuszowski/conjurer/pamiec.json"
SYSTEM_GPT_SETTINGS = "/home/mtuszowski/conjurer/system_gpt_settings.json"
MEMORY_FIVE_MUZYKA = "/home/mtuszowski/conjurer/pamiec_muzyki.json"
MUSIC_FOLDER = "/mnt/g/Muzyka/"
SETTINGS_FILE = "/home/mtuszowski/conjurer/settings.json"
NETRC_FILE = "/home/mtuszowski/.netrc"
LOGSTORE = "/home/mtuszowski/conjurer/logs/"
ACCIDENT_LOG = "/home/mtuszowski/conjurer/accident_log.json"
ENCODING = "utf-8"
GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
DIR_PATH_SADOX = "/mnt/c/Users/mtusz/OneDrive/Dokumenty/Fansadox"
else:
LOGFILE = "/home/pi/Conjurer/discord.log"
MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
NETRC_FILE = "/home/pi/.netrc"
LOGSTORE = "/home/pi/RetroPie/logs/"
ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
ENCODING = "utf-8"
GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
elif platform == "win32":
LOGFILE = "discord.log"
MEMORY_FIVE_SIARA = "pamiec.json"
SYSTEM_GPT_SETTINGS = "system_gpt_settings.json"
MEMORY_FIVE_MUZYKA = "pamiec_muzyki.json"
MUSIC_FOLDER = "G:\\Muzyka\\"
SETTINGS_FILE = "settings.json"
NETRC_FILE = "C:\\Users\\mtusz\\.netrc"
LOGSTORE = "C:\\Users\\mtusz\\OneDrive\\Pulpit\\Conjurer\\logs\\"
ACCIDENT_LOG = "accident_log.json"
ENCODING = "utf-8"
DIR_PATH_SADOX = "C:\\Users\\mtusz\\OneDrive\\Dokumenty\\Fansadox\\"
SEPARATOR_FILE_PATH = "\\"
with open(SETTINGS_FILE, "r", encoding=ENCODING) as f_settings_file:
DATA = json.load(f_settings_file)
+155 -1
View File
@@ -1 +1,155 @@
import bot_music_functions
import music_functions
from constants import MUZYKA
import logging
import discord
from discord.ext import commands
from communication_subroutine import PREPPED_TRACKS
from sys import platform
import re
class MusicModule(commands.Cog):
def __init__(self, bot, logger_name):
self.bot = bot
self.logger = logging.getLogger(logger_name)
@commands.hybrid_command(
name="krecimy_pornola",
description="Wiadomo co, dla kogo i po co",
)
async def krecimy_pornola(self, ctx):
"""
Download a music number from youtube.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. It allows the command to interact
with the Discord API and
"""
await ctx.send("Dej mnie chwilkę")
allowed = False
for role in ctx.author.roles:
if role.name == "Bartender":
allowed = True
if allowed:
self.logger.info("Pornol")
async with ctx.typing():
# wyciagnij linka z kontekstu
content = ctx.message.content.split()
for item_yt in content:
if re.match("http.*", item_yt):
sciezka, files = await music_functions.get_file(ctx, "Pornol", item_yt)
if files:
self.logger.info("Pornol udany")
await ctx.send(f"Jest w tajnym archiwum pod adresem{sciezka})")
else:
await ctx.reply("Jak ładnie szefa poprosisz.")
@commands.hybrid_command(
name="co_na_plejliscie_wariacie",
description="Wyswietl kawalki ktore sa na pocztku list radia",
guild=discord.Object(id=664789470779932693),
)
async def co_na_plejliscie_wariacie(self,ctx):
"""
Download a music number from youtube.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. It allows the command to interact
with the Discord API and
"""
await ctx.send("Dej mnie chwilkę")
async with ctx.typing():
self.logger.info("plejlista")
self.logger.info(PREPPED_TRACKS)
await ctx.send(f"Didżej Hammer i jego sztuczna inteligencja (||he he||) prezentują: \n- obecnie grany jest {(PREPPED_TRACKS['now_playing']).rstrip()} (wg metadanych: {PREPPED_TRACKS['meta']}) \n- następny będzie{(PREPPED_TRACKS['next']).rstrip()} \n- na liście priorytetowej {(PREPPED_TRACKS['priority']).rstrip()} \n- na liście hitów obecnie czeka kawałek {(PREPPED_TRACKS['hit']).rstrip()} \n- na liście wszystkich kawałków {(PREPPED_TRACKS['all']).rstrip()} \n- najświeższym zamówieniem od słuchaczy jest {(PREPPED_TRACKS['requests']).rstrip()} \nZ kolei jingiel to: {(PREPPED_TRACKS['jingles']).rstrip()}")
@commands.hybrid_command(
name="dej_co_z_jutuba",
description="Podaj link do youtube - sciagnie i doda muzyke",
guild=discord.Object(id=664789470779932693),
)
async def dej_co_z_jutuba(self,ctx):
"""
Download a music number from youtube.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. It allows the command to interact
with the Discord API and
"""
await ctx.send("Dej mnie chwilkę")
self.logger.info("Jutub")
async with ctx.typing():
# wyciagnij linka z kontekstu
content = ctx.message.content.split()
for item_yt in content:
if re.match("http.*", item_yt):
_, files = await music_functions.get_file(ctx, "Youtube", item_yt)
if files:
for file_iter in files:
global MUZYKA # pylint: disable=global-variable-not-assigned
MUZYKA["queue"].insert(0, file_iter)
music_functions.MUSIC_FILE_LIST.update_file_list(file_iter)
MUZYKA["requester"].insert(0, ctx.author)
await ctx.send(f"Dodałem do listy {file_iter}")
self.logger.info("Jutub udany")
@commands.hybrid_command(
name="dej_co_ze_spotifaja",
description="Podaj link do spotify - sciagnie i doda muzyke",
guild=discord.Object(id=664789470779932693),
)
async def dej_co_ze_spotifaja(self, ctx):
"""
Get a playlist or a music number from Spotify.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. This parameter is required in all
Discord.py commands
"""
self.logger.info("Spotifaj")
await ctx.send("Dej mnie chwilkę")
async with ctx.typing():
# wyciagnij linka z kontekstu
content = ctx.message.content.split()
for item in content:
if re.match("http.*", item):
dir_path, files = await music_functions.get_file(ctx, "Spotify", item)
if platform == "win32":
separator = "\\"
else:
separator = "/"
for file in files:
file_path = (
dir_path
+ separator
+ file["artist"]
+ " - "
+ file["name"]
+ ".mp3"
)
global MUZYKA # pylint: disable=global-variable-not-assigned
MUZYKA["queue"].insert(
0,
dir_path
+ separator
+ file["artist"]
+ " - "
+ file["name"]
+ ".mp3",
)
MUZYKA["requester"].insert(0, ctx.author)
music_functions.MUSIC_FILE_LIST.update_file_list(file_path)
await ctx.send(
f"Dodałem do listy {dir_path}{file['artist']} - + {file['name']}.mp3"
)
self.logger.info("Spotifaj udany")
async def setup(bot):
logger = logging.getLogger("discord")
music_functions.MUSIC_FILE_LIST.refresh_file_list()
logger.info("Playlist generation")
await bot.add_cog(MusicModule(bot, "discord"))
logger.info("Loading music commands module done")
+247
View File
@@ -0,0 +1,247 @@
import logging
import asyncio
from pathlib import Path, PurePath
from sys import platform
from spotify_dl import spotify
from spotify_dl import youtube as youtube_download
import re
import netrc
import spotipy
from spotipy.oauth2 import SpotifyClientCredentials
from constants import NETRC_FILE, MUSIC_FOLDER, FILE_SERVICE_ADDRESS, GET_MP3, SEND_MP3
import yt_dlp
import requests
netrc_mod = netrc.netrc(NETRC_FILE)
REMOTE_HOST_NAME = "spotipy"
authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME)
logger = logging.getLogger("discord")
spotify_ctrl = spotipy.Spotify(
client_credentials_manager=SpotifyClientCredentials(
client_id=authTokens[0],
client_secret=authTokens[2],
)
)
class MusicFileList(object):
"""
The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a
file service or local directory and update the list with new items.
"""
def __init__(self, uplogger) -> None:
self.music_file_list = []
self.file_service_active = False
self.logger = uplogger
self.logger.info("Created Playlist organizer class")
def refresh_file_list(self):
"""
The `refresh_file_list` function attempts to connect to a file service to retrieve a list of music
files, and if unsuccessful, it populates the list by scanning a local directory for .mp3 files.
"""
try:
self.logger.info("Attempt to connect to file service")
response = requests.get(f"{FILE_SERVICE_ADDRESS}{GET_MP3}", timeout=360)
self.music_file_list = response.json()["music_file_list"]
self.file_service_active = True
except requests.exceptions.RequestException as e:
for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
temp_music_file = mp3_item.as_posix()
if platform == "win32":
temp_music_file = temp_music_file.replace("/", "\\")
self.music_file_list.append(temp_music_file)
self.file_service_active = False
self.logger.error(e.strerror)
self.logger.error("Service Unavailable")
def get_file_list(self):
"""
The `get_file_list` function returns the list of music files stored in the object.
:return: The `music_file_list` attribute is being returned.
"""
return self.music_file_list
def update_file_list(self, item):
"""
The `update_file_list` function appends an item to a music file list and sends a POST request with
the item data to a file service address.
:param item: The `item` parameter in the `update_file_list` method is the file that you want to add
to the `music_file_list` attribute of the class instance. It is then sent as a JSON payload in a
POST request to a file service address along with the endpoint `SEND_MP3`
"""
self.music_file_list.append(item)
post_data = {"item": str(item)}
requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
MUSIC_FILE_LIST = MusicFileList(logger)
async def get_file(ctx, source, link):
"""
Take in three parameters: "ctx" (context),
"source" (a string representing the source of the file), and "link" (a string representing the link
to the file). Execute it in asynchronouse way. And get file from spotify or youtube.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which a command is being executed, including information such as the
message, the channel, the server, and the user who invoked the command
:param source: The source parameter is likely a string that represents the source of the file that
the user wants to retrieve. This could be a website, a cloud storage service, or any other location
where the file is stored
:param link: The "link" parameter is likely a string that represents a URL or file path to the
location of the file that the function is trying to retrieve
"""
item_id = None
item_type = None
file_list = []
if source == "Spotify":
dir_path = None
item_type, item_id = spotify.parse_spotify_url(link)
if item_type in ["album", "track", "playlist"] and item_id:
logger.info(item_type)
logger.info(item_id)
logger.info("Spotify link provided")
file_list = await asyncio.to_thread(
spotify.fetch_tracks, spotify_ctrl, item_type, link
)
logger.info(file_list)
directory_name = await asyncio.to_thread(
spotify.get_item_name, spotify_ctrl, item_type, item_id
)
logger.info(directory_name)
logger.info("Spotify scrape done")
url_data = {"urls": []}
url_dict = {}
url_dict["save_path"] = Path(
PurePath.joinpath(Path(MUSIC_FOLDER), Path(directory_name))
)
url_dict["save_path"].mkdir(parents=True, exist_ok=True)
url_dict["songs"] = file_list
url_data["urls"].append(url_dict.copy())
file_name_f = youtube_download.default_filename
logger.info("YT-DL")
coro = asyncio.to_thread(
youtube_download.download_songs,
songs=url_data,
output_dir=MUSIC_FOLDER,
format_str="bestaudio/best",
skip_mp3=False,
keep_playlist_order=False,
no_overwrites=True,
remove_trailing_tracks="no",
use_sponsorblock="yes",
file_name_f=file_name_f,
multi_core=0,
proxy="",
)
_ = await coro
logger.info("YT-DL done")
dir_path = (url_dict["save_path"].resolve()).as_posix()
if platform == "win32":
dir_path = dir_path.replace("/", "\\")
return dir_path, file_list
await ctx.send(
"No se jaja chyba robisz, Ty byś spotifaja nie poznawszy jakby z krzaków wyskoczył i w dupę kopnął. Plejlista, kawałek albo album prosze."
)
logger.error("Wrong link provided spotify: %s", link)
elif source == "Youtube":
link_ok = False
pat = r"^https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_\+.~#?&\/=]*)$"
if re.match(pat, link) and re.match(".*youtube.*", link):
link_ok = True
if link_ok:
dir_path = ""
file_list = []
query = link
sponsorblock_postprocessor = [
{
"key": "SponsorBlock",
"categories": ["skip_non_music_sections"],
},
{
"key": "ModifyChapters",
"remove_sponsor_segments": ["music_offtopic"],
"force_keyframes": True,
},
]
# TODO: Make sponsorblock work
sponsorblock_postprocessor = []
if platform == "win32":
dir_path = "G:\\Muzyka\\Youtube"
else:
dir_path = "/home/pi/RetroPie/mp3/Youtube"
ydl_opts = {
"proxy": "",
"default_search": "ytsearch",
"format": "bestaudio/best",
"postprocessors": sponsorblock_postprocessor,
"noplaylist": True,
"no_color": False,
"paths": {"home": dir_path},
}
mp3_postprocess_opts = {
"key": "FFmpegExtractAudio",
"preferredcodec": "mp3",
"preferredquality": "192",
}
ydl_opts["postprocessors"].append(mp3_postprocess_opts.copy())
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
try:
coro = asyncio.to_thread(ydl.download, [query])
_ = await coro
except Exception as exce: # pylint: disable=broad-exception-caught
logger.error(exce)
logger.info(
"Failed to download %s, make sure yt_dlp is up to date", link
)
extract = re.search("v=(...........)[&,$]*", link)
ident = extract.group(1)
for item in Path.glob(Path(dir_path), "**/*.mp3"):
file = item.as_posix()
if platform == "win32":
file = file.replace("/", "\\")
if re.match(".*" + ident + ".*", file):
file_list.append(file)
return dir_path, file_list
await ctx.send(
"No se jaja chyba robisz, Ty byś jutuba nie poznawszy jakby z krzaków wyskoczył i w dupę kopnął. Jutuba to nie tuba którą sobie można gdzieś wsadzić."
)
logger.error("Wrong link provided youtube: %s", link)
return "", None
elif source == "Pornol":
dir_path = ""
file_list = []
query = link
sponsorblock_postprocessor = [
{
"key": "SponsorBlock",
"categories": ["skip_non_music_sections"],
},
{
"key": "ModifyChapters",
"remove_sponsor_segments": ["music_offtopic"],
"force_keyframes": True,
},
]
# TODO: Make sponsorblock work
sponsorblock_postprocessor = []
dir_path = "/home/pi/RetroPie/movies/porn"
ydl_opts = {
"postprocessors": sponsorblock_postprocessor,
"paths": {"home": dir_path},
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
try:
coro = asyncio.to_thread(ydl.download, [query])
_ = await coro
except Exception as exce: # pylint: disable=broad-exception-caught
logger.error(exce)
logger.info(
"Failed to download %s, make sure yt_dlp is up to date", link
)
file_list.append("wiadomo co wiadomo gdzie")
return dir_path, file_list
+68
View File
@@ -0,0 +1,68 @@
from discord.ext import commands
import logging
import discord
from typing import Optional
from constants import DATA
historia_fabryczki = DATA["fabryczka"]
class OtherModule(commands.Cog):
def __init__(self, bot, logger_name):
self.bot = bot
self.logger = logging.getLogger(logger_name)
@commands.hybrid_command(
name="przytul", description="Przytul kogoś - daj mention po komendzie :)"
)
async def przytul(ctx, arg: Optional[discord.Member] = None):
"""
Generate a text about hugging mentioned user.
:param ctx: ctx stands for "context" and is a required parameter in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command
:param arg: arg is a parameter of the function "przytul" that expects a Discord member object. The
parameter is optional, meaning that if no member object is provided, it will default to None
:type arg: Optional[discord.Member]
"""
async with ctx.typing():
nieprzytulac = False
for mention in ctx.message.mentions:
for role in mention.roles:
if role.name == "NIEPRZYTULAĆ!":
nieprzytulac = True
if arg and nieprzytulac:
await ctx.send(
f"Żebym ja Ciebie nie przytulił {ctx.message.author.mention}"
)
elif arg:
await ctx.send(
# trunk-ignore(codespell/misspelled)
f"Już dobrze.... Już dobrze... Ojej.. Biedactwo... :( *W ułamku sekundy {arg.mention} znajduje sie w duszącym uścisku. Żebra trzeszczą - kilka pęka. Pacnięcia po plecach grożą odbiciem nerek, a głaskanie po głowie powoduje wstrząs mózgu*"
)
else:
await ctx.send(
"Kogo mam przytulić? *Wyłamuje kostki i przeciąga się - jego 200 kilowa sylwetka złożona z samych mięśni świadczy o tym że jest gotowy*"
)
@commands.hybrid_command(name="fabryczka", description="Historia fabryczki")
async def fabryczka(self,ctx):
"""
Send a general description of "fabryczkagate" to channel.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. The context object provides a way to
interact with the Discord
"""
await ctx.send(historia_fabryczki)
async def setup(bot):
logger = logging.getLogger("discord")
logger.info("Playlist generation")
await bot.add_cog(OtherModule(bot, "discord"))
logger.info("Loading other module done")
+1 -1
View File
@@ -1 +1 @@
import bot_music_functions
import music_functions
+2184
View File
File diff suppressed because it is too large Load Diff