diff --git a/bot.py b/bot.py index afd87be..6924761 100644 --- a/bot.py +++ b/bot.py @@ -122,7 +122,6 @@ class MusicFileList(object): # *=========================================== Predefines -formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") MASTER_TIMEOUT = datetime.now() INITIAL_TIME_WAIT = 500 MUZYKA: Music_Config = {"ctx": None, "queue": [], "requester": []} @@ -243,6 +242,7 @@ handler = handlers.RotatingFileHandler( maxBytes=6 * 1024 * 1024, backupCount=6, ) +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) random.seed() @@ -301,8 +301,13 @@ logger.info("Done") async def on_ready(): """Metoda wywoływana przy połączeniu do serwera.""" logger.debug("%s has connected to Discord!", client.user) + await client.load_extension("voice_recognition") + logger.info("Cogs") + logger.info(client.cogs) await client.change_presence(activity=discord.Game(name="Axe Throwing Darts")) await client.tree.sync() + for com in client.commands: + logger.info(com.qualified_name) check_self.start() check_data_q.start() @@ -537,16 +542,20 @@ async def connect(ctx, arg=None): """ if ctx and arg: logger.info("Ctx and arg defined for connect") - voice_channel = client.get_channel(1060349757349974066) if client.voice_clients: - voice_client = client.voice_clients[0] - if voice_client.is_connected(): + logger.info("Already connected with other client") + else: + vc = None + if ctx.author.voice.channel: + vc = await ctx.author.voice.channel.connect() + else: + voice_channel = client.get_channel(1060349757349974066) + vc = await voice_channel.connect() + if not vc: + logger.error("Not possible to connect to voice") return - voice_channel_connection = await voice_channel.connect() - if not voice_channel_connection: - logger.error("Not possible to connect to voice") - logger.info("Connecting to voice") + logger.info("Connected to voice") async def disconnect(ctx): diff --git a/conjurer_musician/radio_conjurer.liq b/conjurer_musician/radio_conjurer.liq index ec3a6c5..77c4e42 100644 --- a/conjurer_musician/radio_conjurer.liq +++ b/conjurer_musician/radio_conjurer.liq @@ -64,21 +64,23 @@ b = bass_boost(frequency=f, gain=g, s) s = add([s, b]) # Set up interactive control for main volume -a = interactive.float("main_volume", min=0., max=20., 1.) +a = interactive.float("main_volume", min=0., max=20., 0.4) s = compress.multiband.interactive(bands=7, s) +mic_gain = interactive.float("mic_volume", min=0., max=20., 6.) +# Apply audio processing effects +tmic = buffer(input.pulseaudio()) # Microphone +mic = amplify(mic_gain, tmic) +mic = gate(threshold=-80., range=-120., mic) +mic = compress(threshold=0., ratio=2.,mic) +mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic) + +s = add([s, mic]) + # Apply audio processing effects s = nrj(normalize(s)) s = amplify(a, s) -mic = buffer(input.pulseaudio()) # Microphone -mic = gate(threshold=-30., range=-80., mic) -mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic) - -s = add([s, mic]) -#w = interactive.float("wetness", min=0., max=1., 1.) -#s = dry_wet(w,s,s2) - # Skip blank sections in the stream s = blank.skip(max_blank=10., s) diff --git a/install_main_bot.sh b/install_main_bot.sh index ddac526..4eb0f03 100755 --- a/install_main_bot.sh +++ b/install_main_bot.sh @@ -1,4 +1,5 @@ #!/bin/bash +sudo apt-get install python3-dev cd /home/pi mdkir Conjurer cd Conjurer @@ -9,6 +10,11 @@ source ./env/bin/activate ./env/bin/python3 -m pip install -r requirements_bot.txt sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py +sed -i '1i\import logging' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py + +sed -i '21i\ logger = logging.getLogger("discord")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py +sed -i '22i\ logger.info("Playlist")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py + deactivate sudo cp ./conjurer.service /etc/systemd/system/ sudo systemctl daemon-reload diff --git a/install_music_bot.sh b/install_music_bot.sh index 210c6c5..3d1364a 100755 --- a/install_music_bot.sh +++ b/install_music_bot.sh @@ -32,3 +32,7 @@ sudo cp /home/pi/conjurer/conjurer_musician/radio_service.service /etc/systemd/s sudo systemctl daemon-reload sudo systemctl start radio_service.service sudo systemctl enable radio_service.service + + +sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py +sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py diff --git a/requirements_bot.txt b/requirements_bot.txt index 7c8fed6..dd2fcd4 100644 --- a/requirements_bot.txt +++ b/requirements_bot.txt @@ -13,4 +13,7 @@ tiktoken PyNaCl flask waitress -clickupython \ No newline at end of file +clickupython +assemblyai[extras] +SpeechRecognition +git+https://github.com/imayhaveborkedit/discord-ext-voice-recv \ No newline at end of file diff --git a/sr_example.py b/sr_example.py new file mode 100644 index 0000000..3c176cf --- /dev/null +++ b/sr_example.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- + +from __future__ import annotations + +import logging + +from discord.ext.voice_recv.sinks import AudioSink + +log = logging.getLogger(__name__) + +__all__ = [ + 'SpeechRecognitionSink', +] + +try: + import discord.ext.speech_recognition as sr # type: ignore +except ImportError: + + def SpeechRecognitionSink(**kwargs) -> AudioSink: + """A stub for when the SpeechRecognition module isn't found.""" + raise RuntimeError('The SpeechRecognition module is required to use this sink.') + +else: + import time + import array + import asyncio + import audioop + + from collections import defaultdict + + from discord.ext.voice_recv.sinks import SilencePacket + + from typing import TYPE_CHECKING, TypedDict + + if TYPE_CHECKING: + from concurrent.futures import Future as CFuture + from typing import Literal, Callable, Optional, Any, Final, Protocol, Awaitable, TypeVar + + from discord import Member + + from ..opus import VoiceData + from ..types import MemberOrUser as User + + T = TypeVar('T') + + SRRecognizerMethod = Literal[ + 'sphinx', + 'google', + 'google_cloud', + 'wit', + 'azure', + 'bing', + 'lex', + 'houndify', + 'amazon', + 'assemblyai', + 'ibm', + 'tensorflow', + 'whisper', + 'vosk', + ] + + class SRStopper(Protocol): + def __call__(self, wait: bool = True, /) -> None: + ... + + SRProcessDataCB = Callable[[sr.Recognizer, sr.AudioData, User], Optional[str]] + SRTextCB = Callable[[User, str], Any] + + class _StreamData(TypedDict): + stopper: Optional[SRStopper] + recognizer: sr.Recognizer + buffer: array.array[int] + + class SpeechRecognitionSink(AudioSink): # type: ignore + def __init__( + self, + *, + process_cb: Optional[SRProcessDataCB] = None, + text_cb: Optional[SRTextCB] = None, + default_recognizer: SRRecognizerMethod = 'assemblyai', + phrase_time_limit: int = 10, + ignore_silence_packets: bool = True, + ): + super().__init__(None) + self.process_cb: Optional[SRProcessDataCB] = process_cb + self.text_cb: Optional[SRTextCB] = text_cb + self.phrase_time_limmit: int = phrase_time_limit + self.ignore_silence_packets: bool = ignore_silence_packets + + self.default_recognizer: SRRecognizerMethod = default_recognizer + self._stream_data: defaultdict[int, _StreamData] = defaultdict( + lambda: _StreamData(stopper=None, recognizer=sr.Recognizer(), buffer=array.array('B')) + ) + + def _await(self, coro: Awaitable[T]) -> CFuture[T]: + assert self.client is not None + return asyncio.run_coroutine_threadsafe(coro, self.client.loop) + + def wants_opus(self) -> bool: + return False + + def write(self, user: Optional[User], data: VoiceData) -> None: + if self.ignore_silence_packets and isinstance(data.packet, SilencePacket): + return + + if user is None: + return + + sdata = self._stream_data[user.id] + sdata['buffer'].extend(data.pcm) + + if not sdata['stopper']: + sdata['stopper'] = sdata['recognizer'].listen_in_background( + DiscordSRAudioSource(sdata['buffer']), self.background_listener(user), self.phrase_time_limmit + ) + + def background_listener(self, user: User): + process_cb = self.process_cb or self.get_default_process_callback() + text_cb = self.text_cb or self.get_default_text_callback() + + def callback(_recognizer: sr.Recognizer, _audio: sr.AudioData): + output = process_cb(_recognizer, _audio, user) + if output is not None: + text_cb(user, output) + + return callback + + def get_default_process_callback(self) -> SRProcessDataCB: + def cb(recognizer: sr.Recognizer, audio: sr.AudioData, user: Optional[User]) -> Optional[str]: + log.debug("Got %s, %s, %s", audio, audio.sample_rate, audio.sample_width) + text: Optional[str] = None + try: + # they changed recognize_google to be optionally assigned at runtime... + func = getattr(recognizer, 'recognize_' + self.default_recognizer, recognizer.recognize_google) # type: ignore + text = func(audio) # type: ignore + except sr.UnknownValueError: + log.debug("Bad speech chunk") + # self._debug_audio_chunk(audio) + + return text + + return cb + + def get_default_text_callback(self) -> SRTextCB: + def cb(user: Optional[User], text: Optional[str]) -> Any: + log.info("%s said: %s", user.display_name if user else 'Someone', text) + + return cb + + @AudioSink.listener() + def on_voice_member_disconnect(self, member: Member, ssrc: Optional[int]) -> None: + self._drop(member.id) + + def cleanup(self) -> None: + for user_id in tuple(self._stream_data.keys()): + self._drop(user_id) + + def _drop(self, user_id: int) -> None: + data = self._stream_data.pop(user_id) + + stopper = data.get('stopper') + if stopper: + stopper() + + buffer = data.get('buffer') + if buffer: + # arrays don't have a clear function + del buffer[:] + + def _debug_audio_chunk(self, audio: sr.AudioData, filename: str = 'sound.wav') -> None: + import io, wave, discord + + with io.BytesIO() as b: + with wave.open(b, 'wb') as writer: + writer.setframerate(48000) + writer.setsampwidth(2) + writer.setnchannels(2) + writer.writeframes(audio.get_wav_data()) + + b.seek(0) + f = discord.File(b, filename) + self._await(self.voice_client.channel.send(file=f)) # type: ignore + + class DiscordSRAudioSource(sr.AudioSource): + little_endian: Final[bool] = True + SAMPLE_RATE: Final[int] = 48_000 + SAMPLE_WIDTH: Final[int] = 2 + CHANNELS: Final[int] = 2 + CHUNK: Final[int] = 960 + + def __init__(self, buffer: array.array[int]): + self.buffer = buffer + self._entered: bool = False + + @property + def stream(self): + return self + + def __enter__(self): + if self._entered: + log.warning('Already entered sr audio source') + self._entered = True + return self + + def __exit__(self, *exc) -> None: + self._entered = False + if any(exc): + log.exception('Error closing sr audio source') + + def read(self, size: int) -> bytes: + # TODO: make this timeout configurable + for _ in range(10): + if len(self.buffer) < size * self.CHANNELS: + time.sleep(0.1) + else: + break + else: + if len(self.buffer) == 0: + return b'' + + chunksize = size * self.CHANNELS + audiochunk = self.buffer[:chunksize].tobytes() + del self.buffer[: min(chunksize, len(audiochunk))] + audiochunk = audioop.tomono(audiochunk, 2, 1, 1) + return audiochunk + + def close(self) -> None: + self.buffer.clear() \ No newline at end of file diff --git a/stream_test.py b/stream_test.py new file mode 100644 index 0000000..82a410e --- /dev/null +++ b/stream_test.py @@ -0,0 +1,62 @@ +# Start by making sure the `assemblyai` package is installed. +# If not, you can install it by running the following command: +# pip install -U assemblyai +# +# Then, make sure you have PyAudio installed: https://pypi.org/project/PyAudio/ +# +# Note: Some macOS users might need to use `pip3` instead of `pip`. + +import pyaudio +import assemblyai as aai + +aai.settings.api_key = "aa9962f0088a449a9c4ab2361e96cc08" + +def on_open(session_opened: aai.RealtimeSessionOpened): + "This function is called when the connection has been established." + + print("Session ID:", session_opened.session_id) + +def on_data(transcript: aai.RealtimeTranscript): + "This function is called when a new transcript has been received." + + if not transcript.text: + return + + if isinstance(transcript, aai.RealtimeFinalTranscript): + print(transcript.text, end="\r\n") + else: + print(transcript.text, end="\r") + +def on_error(error: aai.RealtimeError): + "This function is called when the connection has been closed." + + print("An error occured:", error) + +def on_close(): + "This function is called when the connection has been closed." + + print("Closing Session") + +transcriber = aai.RealtimeTranscriber( + on_data=on_data, + on_error=on_error, + sample_rate=44_100, + on_open=on_open, # optional + on_close=on_close, # optional +) + + +pa = pyaudio.PyAudio() +for i in range(pa.get_device_count()): + print (pa.get_device_info_by_index(i)) + +# Start the connection +#transcriber.connect() + +# Open a microphone stream +#microphone_stream = aai.extras.MicrophoneStream() + +# Press CTRL+C to abort +#transcriber.stream(microphone_stream) + +#transcriber.close() diff --git a/test_time.py b/test_time.py new file mode 100644 index 0000000..63b1f06 --- /dev/null +++ b/test_time.py @@ -0,0 +1,7 @@ +import time +first_time = time.time_ns() + +time.sleep(1) +time_diff = time.time_ns() - first_time +print(time_diff) +#2000149433 \ No newline at end of file diff --git a/voice_example.py b/voice_example.py new file mode 100644 index 0000000..36cec22 --- /dev/null +++ b/voice_example.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +import discord +from discord.ext import commands, voice_recv + +discord.opus._load_default() + +bot = commands.Bot(command_prefix=commands.when_mentioned, intents=discord.Intents.all()) + +class Testing(commands.Cog): + def __init__(self, bot): + self.bot = bot + + @commands.command() + async def test(self, ctx): + def callback(user, data: voice_recv.VoiceData): + print(f"Got packet from {user}") + + ## voice power level, how loud the user is speaking + # ext_data = packet.extension_data.get(voice_recv.ExtensionID.audio_power) + # value = int.from_bytes(ext_data, 'big') + # power = 127-(value & 127) + # print('#' * int(power * (79/128))) + ## instead of 79 you can use shutil.get_terminal_size().columns-1 + + vc = await ctx.author.voice.channel.connect(cls=voice_recv.VoiceRecvClient) + vc.listen(voice_recv.BasicSink(callback)) + + @commands.command() + async def stop(self, ctx): + await ctx.voice_client.disconnect() + + @commands.command() + async def die(self, ctx): + ctx.voice_client.stop() + await ctx.bot.close() + +@bot.event +async def on_ready(): + print('Logged in as {0.id}/{0}'.format(bot.user)) + print('------') + await bot.add_cog(Testing(bot)) + +REMOTE_HOST_NAME = "discord" + + +bot.run("NzUxNDQxODgzNDA2MDA4NDEw.Go--wb.Nr28Oo4eYAP1p2XFVHF5uIQfcPD7s_jO2NfMCQ") diff --git a/voice_recognition.py b/voice_recognition.py new file mode 100644 index 0000000..73bdfb8 --- /dev/null +++ b/voice_recognition.py @@ -0,0 +1,233 @@ +import logging +import wave +import asyncio +import assemblyai as aai +import discord +from discord.ext import commands, voice_recv, tasks +from discord.opus import Decoder as OpusDecoder +import time +# Replace with your API key +aai.settings.api_key = "aa9962f0088a449a9c4ab2361e96cc08" +discord.opus._load_default() +CHANNELS = OpusDecoder.CHANNELS +SAMPLE_WIDTH = OpusDecoder.SAMPLE_SIZE // OpusDecoder.CHANNELS +SAMPLING_RATE = OpusDecoder.SAMPLING_RATE + +#rotate file after there is 0.5s between last received pcm for user. +#delete messsages after user disconnect + +logger = logging.getLogger("discord") +location = "/home/pi/Conjurer/transcripts/" + +class CommunicationObject: + def __init__(self, msg_type, user, data): + self.type = msg_type + self.user = user + self.data = data + def __repr__(self): + return f"{self.type} : {self.user} : {self.data}" + +class WaveWriter: + def __init__(self, user_id, queue): + self.queue = queue + self.user = user_id + self.username = str(user_id.id) + #here we can add hashing function to make transcription files not possible to be connected with discord id + self.present_file_id = 0 + self.file_name_past = None + self.file_name_present = location + self.username + "_" + str(self.present_file_id) + ".mp3" + self.transcript_file_present : wave.Wave_write = wave.open( + self.file_name_present, "wb" + ) + self.transcript_file_present.setnchannels(CHANNELS) + self.transcript_file_present.setsampwidth(SAMPLE_WIDTH) + self.transcript_file_present.setframerate(SAMPLING_RATE) + self.file_name_future = location + self.username + "_" + str(self.present_file_id + 1) + ".mp3" + self.transcript_file_future : wave.Wave_write = wave.open( + self.file_name_future, "wb" + ) + self.transcript_file_future.setnchannels(CHANNELS) + self.transcript_file_future.setsampwidth(SAMPLE_WIDTH) + self.transcript_file_future.setframerate(SAMPLING_RATE) + + async def rotate(self): + + self.transcript_file_present.close() + self.file_name_past = self.file_name_present + if self.present_file_id > 10: + self.present_file_id = 0 + else: + self.present_file_id += 1 + self.file_name_present = self.file_name_future + self.transcript_file_present = self.transcript_file_future + self.file_name_future = location + self.username + "_" + str(self.present_file_id + 1) + ".mp3" + self.transcript_file_future : wave.Wave_write = wave.open( + self.file_name_future, "wb" + ) + self.transcript_file_future.setnchannels(CHANNELS) + self.transcript_file_future.setsampwidth(SAMPLE_WIDTH) + self.transcript_file_future.setframerate(SAMPLING_RATE) + operation = CommunicationObject(msg_type="send_file",user=self.user,data=[self.file_name_past]) + + await self.queue.put(operation) + + def writeframes(self, pcmdata): + self.transcript_file_present.writeframes(pcmdata) + + def cleanup(self): + logger.info("Cleanup for user %s", self.username) + self.transcript_file_present.close() + self.transcript_file_future.close() + + +class SRBuffer(voice_recv.AudioSink): + """Endpoint AudioSink that generates a wav file. + Best used in conjunction with a silence generating sink. (TBD) + """ + # on member join dodajemytypa do listy + # on member disconnect - dropujemy go + def __init__(self, queue): + super().__init__() + self.queue = queue + self.wavewriter = {} + + def on_user_connect(self, username): + self.wavewriter[str(username.id)] = [WaveWriter(username, self.queue), time.time_ns(), time.time_ns(), False, False] + + def on_user_disconnect(self,username): + self.wavewriter[str(username.id)].cleanup() + self.wavewriter.pop(str(username.id)) + + def wants_opus(self) -> bool: + return False + + def write(self, user, data) -> None: + #logger.info("DAta write") + if user: + self.wavewriter[str(user.id)][0].writeframes(data.pcm) + self.wavewriter[str(user.id)][1] = time.time_ns() #time from last write + self.wavewriter[str(user.id)][3] = True #data written in last iter + self.wavewriter[str(user.id)][4] = True #data in buff + time.sleep(0.001) + + def cleanup(self) -> None: + try: + logger.info("Cleanup for SRBuffer") + for item in self.wavewriter.values(): + item[0].cleanup() + except Exception: + logger.warning("WaveSink got error closing file on cleanup", exc_info=True) + + +class Transcriber(commands.Cog): + def __init__(self, bot): + self.bot = bot + self.threads = [] + self.comm_queue = asyncio.Queue() + self.wsink = SRBuffer(self.comm_queue) + self.worker = None + self.mc = None + self.vc = None + + @commands.hybrid_command(name="transcribe") + async def test(self, ctx): + if self.vc: + vc = self.vc #to juz powinien byc voice channel z funkcja conenct + else: + vc = None + self.mc = ctx.message.channel + if self.bot.voice_clients: + if isinstance(self.bot.voice_clients[0], voice_recv.VoiceRecvClient): + logger.debug("Already transcribing") + else: + logger.debug("Already connected with other client") + else: + logger.debug("Connected") + vc = await ctx.author.voice.channel.connect(cls=voice_recv.VoiceRecvClient) + logger.info(self.bot.voice_clients) + self.check_data.start() + self.worker = asyncio.create_task(self.transcribe_output_queue()) + vc.listen(self.wsink) + + + @tasks.loop(seconds=0.5) + async def check_data(self): + for item in self.wsink.wavewriter.values(): + if not item[3]: + timediff_rotation = time.time_ns() - item[2] + timediff_write = time.time_ns() -item[1] + if timediff_rotation > 13000149433 and timediff_write > 500014943 and item[4]: + logger.info("File rotation time since last write %s %s", timediff_write, timediff_write/1e9) + logger.info("File rotation time since last rotation %s %s",timediff_rotation, timediff_rotation/1e9) + item[2] = time.time_ns() + item[4] = False + await item[0].rotate() + item[3] = False + + @commands.Cog.listener() + async def on_voice_state_update(self, user, before, after): + if user == self.bot.user: + logger.debug("Ignoring self") + return + + if before.channel is None: + logger.info("User %s connected to channel %s", user, after.channel.name) + self.wsink.on_user_connect(user) + elif after.channel is None: + logger.info("User %s disconnected from channel %s", user, before.channel.name) + operation = CommunicationObject(msg_type="user_cleanup", user=user, data=None) + await self.comm_queue.put(operation) + else: + logger.debug("User VC status changed %s", user.id) + logger.debug("Before %s", before) + logger.debug("After %s", after) + + + @commands.command(name="stop_transcribe") + async def stop(self, ctx): + self.check_data.stop() + stop_token = CommunicationObject("STOP",None,None) + + await self.comm_queue.put(stop_token) + await ctx.voice_client.disconnect() + + async def transcribe_output_queue(self): + logger.info("Transcript uploader start") + config = aai.TranscriptionConfig(language_code="pl") + transcriber = aai.Transcriber() + logger.debug("Transcriber queue id: %s",id(self.comm_queue)) + + while True: + logger.debug("waiting for tasks") + item = await self.comm_queue.get() + logger.debug("Got %s", item) + if "STOP" in item.type: + logger.debug("Queue ended") + break + elif "send_file" in item.type: + logger.debug("Sending file for transcription") + transcript = None + for iter in item.data: + try: + coro = asyncio.to_thread( + transcriber.transcribe, + iter, + config=config + ) + transcript = await coro + except Exception as e: + logger.warn("Exceptiom occured %s", e) + await self.mc.send(f"{item.user} : {transcript.text}") + if transcript.error: + logger.error(transcript.error) + raise AssertionError + elif "user_cleanup" in item.type: + logger.info("User %s disconnected - cleanup action") + else: + logger.warn("Something went wrong with object %s", item) + logger.info("Transcript uploader stoped") + +async def setup(bot): + logger = logging.getLogger("discord") + await bot.add_cog(Transcriber(bot)) + logger.info("Loading voice transcribed module done")