From 7857226e129ec3ec1e0fec8aefb4c028ab703bcc Mon Sep 17 00:00:00 2001 From: mtuszowski Date: Fri, 23 Aug 2024 18:46:40 +0200 Subject: [PATCH] Fix --- conjurer_musician/radio_conjurer.liq | 20 +-- install_main_bot.sh | 5 + install_music_bot.sh | 4 + sr_example.py | 229 +++++++++++++++++++++++++++ voice_recognition.py | 26 ++- 5 files changed, 274 insertions(+), 10 deletions(-) create mode 100644 sr_example.py diff --git a/conjurer_musician/radio_conjurer.liq b/conjurer_musician/radio_conjurer.liq index ec3a6c5..77c4e42 100644 --- a/conjurer_musician/radio_conjurer.liq +++ b/conjurer_musician/radio_conjurer.liq @@ -64,21 +64,23 @@ b = bass_boost(frequency=f, gain=g, s) s = add([s, b]) # Set up interactive control for main volume -a = interactive.float("main_volume", min=0., max=20., 1.) +a = interactive.float("main_volume", min=0., max=20., 0.4) s = compress.multiband.interactive(bands=7, s) +mic_gain = interactive.float("mic_volume", min=0., max=20., 6.) +# Apply audio processing effects +tmic = buffer(input.pulseaudio()) # Microphone +mic = amplify(mic_gain, tmic) +mic = gate(threshold=-80., range=-120., mic) +mic = compress(threshold=0., ratio=2.,mic) +mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic) + +s = add([s, mic]) + # Apply audio processing effects s = nrj(normalize(s)) s = amplify(a, s) -mic = buffer(input.pulseaudio()) # Microphone -mic = gate(threshold=-30., range=-80., mic) -mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic) - -s = add([s, mic]) -#w = interactive.float("wetness", min=0., max=1., 1.) -#s = dry_wet(w,s,s2) - # Skip blank sections in the stream s = blank.skip(max_blank=10., s) diff --git a/install_main_bot.sh b/install_main_bot.sh index ddac526..6bf447e 100755 --- a/install_main_bot.sh +++ b/install_main_bot.sh @@ -9,6 +9,11 @@ source ./env/bin/activate ./env/bin/python3 -m pip install -r requirements_bot.txt sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py +sed -i '1i\import logging' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py + +sed -i '21i\ logger = logging.getLogger("discord")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py +sed -i '22i\ logger.info("Playlist")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py + deactivate sudo cp ./conjurer.service /etc/systemd/system/ sudo systemctl daemon-reload diff --git a/install_music_bot.sh b/install_music_bot.sh index 210c6c5..3d1364a 100755 --- a/install_music_bot.sh +++ b/install_music_bot.sh @@ -32,3 +32,7 @@ sudo cp /home/pi/conjurer/conjurer_musician/radio_service.service /etc/systemd/s sudo systemctl daemon-reload sudo systemctl start radio_service.service sudo systemctl enable radio_service.service + + +sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py +sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py diff --git a/sr_example.py b/sr_example.py new file mode 100644 index 0000000..1362457 --- /dev/null +++ b/sr_example.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- + +from __future__ import annotations + +import logging + +from discord.ext.voice_recv.sinks import AudioSink + +log = logging.getLogger(__name__) + +__all__ = [ + 'SpeechRecognitionSink', +] + +try: + import discord.ext.speech_recognition as sr # type: ignore +except ImportError: + + def SpeechRecognitionSink(**kwargs) -> AudioSink: + """A stub for when the SpeechRecognition module isn't found.""" + raise RuntimeError('The SpeechRecognition module is required to use this sink.') + +else: + import time + import array + import asyncio + import audioop + + from collections import defaultdict + + from discord.ext.voice_recv.sinks import SilencePacket + + from typing import TYPE_CHECKING, TypedDict + + if TYPE_CHECKING: + from concurrent.futures import Future as CFuture + from typing import Literal, Callable, Optional, Any, Final, Protocol, Awaitable, TypeVar + + from discord import Member + + from ..opus import VoiceData + from ..types import MemberOrUser as User + + T = TypeVar('T') + + SRRecognizerMethod = Literal[ + 'sphinx', + 'google', + 'google_cloud', + 'wit', + 'azure', + 'bing', + 'lex', + 'houndify', + 'amazon', + 'assemblyai', + 'ibm', + 'tensorflow', + 'whisper', + 'vosk', + ] + + class SRStopper(Protocol): + def __call__(self, wait: bool = True, /) -> None: + ... + + SRProcessDataCB = Callable[[sr.Recognizer, sr.AudioData, User], Optional[str]] + SRTextCB = Callable[[User, str], Any] + + class _StreamData(TypedDict): + stopper: Optional[SRStopper] + recognizer: sr.Recognizer + buffer: array.array[int] + + class SpeechRecognitionSink(AudioSink): # type: ignore + def __init__( + self, + *, + process_cb: Optional[SRProcessDataCB] = None, + text_cb: Optional[SRTextCB] = None, + default_recognizer: SRRecognizerMethod = 'google', + phrase_time_limit: int = 10, + ignore_silence_packets: bool = True, + ): + super().__init__(None) + self.process_cb: Optional[SRProcessDataCB] = process_cb + self.text_cb: Optional[SRTextCB] = text_cb + self.phrase_time_limmit: int = phrase_time_limit + self.ignore_silence_packets: bool = ignore_silence_packets + + self.default_recognizer: SRRecognizerMethod = default_recognizer + self._stream_data: defaultdict[int, _StreamData] = defaultdict( + lambda: _StreamData(stopper=None, recognizer=sr.Recognizer(), buffer=array.array('B')) + ) + + def _await(self, coro: Awaitable[T]) -> CFuture[T]: + assert self.client is not None + return asyncio.run_coroutine_threadsafe(coro, self.client.loop) + + def wants_opus(self) -> bool: + return False + + def write(self, user: Optional[User], data: VoiceData) -> None: + if self.ignore_silence_packets and isinstance(data.packet, SilencePacket): + return + + if user is None: + return + + sdata = self._stream_data[user.id] + sdata['buffer'].extend(data.pcm) + + if not sdata['stopper']: + sdata['stopper'] = sdata['recognizer'].listen_in_background( + DiscordSRAudioSource(sdata['buffer']), self.background_listener(user), self.phrase_time_limmit + ) + + def background_listener(self, user: User): + process_cb = self.process_cb or self.get_default_process_callback() + text_cb = self.text_cb or self.get_default_text_callback() + + def callback(_recognizer: sr.Recognizer, _audio: sr.AudioData): + output = process_cb(_recognizer, _audio, user) + if output is not None: + text_cb(user, output) + + return callback + + def get_default_process_callback(self) -> SRProcessDataCB: + def cb(recognizer: sr.Recognizer, audio: sr.AudioData, user: Optional[User]) -> Optional[str]: + log.debug("Got %s, %s, %s", audio, audio.sample_rate, audio.sample_width) + text: Optional[str] = None + try: + # they changed recognize_google to be optionally assigned at runtime... + func = getattr(recognizer, 'recognize_' + self.default_recognizer, recognizer.recognize_google) # type: ignore + text = func(audio) # type: ignore + except sr.UnknownValueError: + log.debug("Bad speech chunk") + # self._debug_audio_chunk(audio) + + return text + + return cb + + def get_default_text_callback(self) -> SRTextCB: + def cb(user: Optional[User], text: Optional[str]) -> Any: + log.info("%s said: %s", user.display_name if user else 'Someone', text) + + return cb + + @AudioSink.listener() + def on_voice_member_disconnect(self, member: Member, ssrc: Optional[int]) -> None: + self._drop(member.id) + + def cleanup(self) -> None: + for user_id in tuple(self._stream_data.keys()): + self._drop(user_id) + + def _drop(self, user_id: int) -> None: + data = self._stream_data.pop(user_id) + + stopper = data.get('stopper') + if stopper: + stopper() + + buffer = data.get('buffer') + if buffer: + # arrays don't have a clear function + del buffer[:] + + def _debug_audio_chunk(self, audio: sr.AudioData, filename: str = 'sound.wav') -> None: + import io, wave, discord + + with io.BytesIO() as b: + with wave.open(b, 'wb') as writer: + writer.setframerate(48000) + writer.setsampwidth(2) + writer.setnchannels(2) + writer.writeframes(audio.get_wav_data()) + + b.seek(0) + f = discord.File(b, filename) + self._await(self.voice_client.channel.send(file=f)) # type: ignore + + class DiscordSRAudioSource(sr.AudioSource): + little_endian: Final[bool] = True + SAMPLE_RATE: Final[int] = 48_000 + SAMPLE_WIDTH: Final[int] = 2 + CHANNELS: Final[int] = 2 + CHUNK: Final[int] = 960 + + def __init__(self, buffer: array.array[int]): + self.buffer = buffer + self._entered: bool = False + + @property + def stream(self): + return self + + def __enter__(self): + if self._entered: + log.warning('Already entered sr audio source') + self._entered = True + return self + + def __exit__(self, *exc) -> None: + self._entered = False + if any(exc): + log.exception('Error closing sr audio source') + + def read(self, size: int) -> bytes: + # TODO: make this timeout configurable + for _ in range(10): + if len(self.buffer) < size * self.CHANNELS: + time.sleep(0.1) + else: + break + else: + if len(self.buffer) == 0: + return b'' + + chunksize = size * self.CHANNELS + audiochunk = self.buffer[:chunksize].tobytes() + del self.buffer[: min(chunksize, len(audiochunk))] + audiochunk = audioop.tomono(audiochunk, 2, 1, 1) + return audiochunk + + def close(self) -> None: + self.buffer.clear() \ No newline at end of file diff --git a/voice_recognition.py b/voice_recognition.py index b543873..b208080 100644 --- a/voice_recognition.py +++ b/voice_recognition.py @@ -6,9 +6,10 @@ import assemblyai as aai import discord -from discord.ext import commands +from discord.ext import commands, voice_recv # Replace with your API key aai.settings.api_key = "aa9962f0088a449a9c4ab2361e96cc08" +discord.opus._load_default() # URL of the file to transcribe FILE_URL = "https://github.com/AssemblyAI-Community/audio-examples/raw/main/20230607_me_canadian_wildfires.mp3" @@ -49,6 +50,29 @@ class Transcriber(commands.Cog): ) for utterance in transcript.utterances: print(f"Speaker {utterance.speaker}: {utterance.text}") + @commands.hybrid_command(name="test") + async def test(self, ctx): + def callback(user, data: voice_recv.VoiceData): + print(f"Got packet from {user}") + + ## voice power level, how loud the user is speaking + # ext_data = packet.extension_data.get(voice_recv.ExtensionID.audio_power) + # value = int.from_bytes(ext_data, 'big') + # power = 127-(value & 127) + # print('#' * int(power * (79/128))) + ## instead of 79 you can use shutil.get_terminal_size().columns-1 + + vc = await ctx.author.voice.channel.connect(cls=voice_recv.VoiceRecvClient) + vc.listen(voice_recv.BasicSink(callback)) + + @commands.command() + async def stop(self, ctx): + await ctx.voice_client.disconnect() + + @commands.command() + async def die(self, ctx): + ctx.voice_client.stop() + await ctx.bot.close() async def setup(bot): await bot.add_cog(Transcriber(bot)) \ No newline at end of file