fix: never swallow task exceptions - diagnose WHY the bot died

The restart-loop with only 'Starting discord bot' in the logs was caused by
gather(return_exceptions=True) eating the client.start() exception: a failed
login looked like a clean exit and docker just restarted the container.

bot.py:
- _run_bot: catch discord.LoginFailure / PrivilegedIntentsRequired with
  CRITICAL messages telling exactly what to fix (token entry / Developer
  Portal intents), full traceback for anything else, then re-raise
- main() returns an exit code; every task exception is re-logged as the
  LAST line in docker logs ('Task died: ...'); process exits 1 on failure
- log the token source (env vs netrc) and its length at startup to catch
  truncated/wrong-field netrc entries
- _run_comm_subroutine: comm crash is logged loudly but no longer relies on
  gather to surface it (bot keeps running - Discord side is independent)

communication_subroutine.py (crash found by the new stub tests):
- comm_subroutine used 'logger' before its local assignment
  (UnboundLocalError killed the whole comm layer at every startup)
- re-comment the flask_debug thread: it binds the same host:port as
  waitress, so running both dies with 'address in use'

Verified via stub-injected client: LoginFailure -> CRITICAL + exit 1,
PrivilegedIntentsRequired -> CRITICAL + exit 1, unknown exception -> full
traceback + exit 1, clean shutdown -> exit 0; comm layer no longer crashes.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Michal Tuszowski
2026-07-08 00:22:55 +02:00
committed by Michał Tuszowski
parent 020a6b114a
commit 460cf072ad
2 changed files with 64 additions and 8 deletions
+58 -5
View File
@@ -216,19 +216,62 @@ async def on_ready():
async def _run_comm_subroutine(stop_event: threading.Event) -> None: async def _run_comm_subroutine(stop_event: threading.Event) -> None:
"""Run the blocking comm subroutine in a worker thread.""" """Run the blocking comm subroutine in a worker thread.
A crash here takes down the internal HTTP endpoints only - the Discord
side keeps running, so log loudly and swallow.
"""
try:
await asyncio.to_thread(comm_subroutine, stop_event) await asyncio.to_thread(comm_subroutine, stop_event)
except Exception: # pylint: disable=broad-exception-caught
logger.exception(
"Comm layer crashed - internal HTTP endpoints are down "
"(musician/librarian callbacks will not arrive); bot continues"
)
async def _run_bot(token: str, shutdown_event: asyncio.Event) -> None: async def _run_bot(token: str, shutdown_event: asyncio.Event) -> None:
"""Start the Discord client and flag shutdown when it returns.""" """Start the Discord client; log WHY it died before flagging shutdown.
Exceptions were previously swallowed by ``gather(return_exceptions=True)``
which made a failed login look like a clean exit (silent crash-loop in
docker). Now every death is diagnosed on the console first.
"""
try: try:
await client.start(token, log_handler=None) await client.start(token, log_handler=None)
except discord.LoginFailure:
logger.critical(
"FATAL: Discord REJECTED the token (Improper token). Fix the "
"'discord' entry in the netrc mounted at CONJURER_NETRC_FILE or "
"the DISCORD_TOKEN env var. If the token leaked/reset, generate a "
"new one in the Discord Developer Portal -> Bot -> Reset Token."
)
raise
except discord.PrivilegedIntentsRequired:
logger.critical(
"FATAL: this bot application does not have the Privileged Gateway "
"Intents enabled. Open Discord Developer Portal -> your app -> "
"Bot -> enable 'Presence', 'Server Members' and 'Message Content' "
"Intents, then restart."
)
raise
except Exception: # pylint: disable=broad-exception-caught
logger.exception("FATAL: Discord client crashed at startup/runtime")
raise
finally: finally:
shutdown_event.set() shutdown_event.set()
async def main() -> None: async def main() -> int:
"""Run both halves; return a process exit code (0 = clean shutdown)."""
if TOKEN:
token_source = (
"env DISCORD_TOKEN" if os.getenv("DISCORD_TOKEN") else "netrc file"
)
logger.info(
"Discord token: loaded from %s (length %d)", token_source, len(TOKEN)
)
logger.info("Starting discord bot") logger.info("Starting discord bot")
shutdown_event = asyncio.Event() shutdown_event = asyncio.Event()
comm_stop_event = threading.Event() comm_stop_event = threading.Event()
@@ -236,6 +279,7 @@ async def main() -> None:
comm_task = asyncio.create_task(_run_comm_subroutine(comm_stop_event)) comm_task = asyncio.create_task(_run_comm_subroutine(comm_stop_event))
bot_task = asyncio.create_task(_run_bot(TOKEN, shutdown_event)) bot_task = asyncio.create_task(_run_bot(TOKEN, shutdown_event))
exit_code = 0
try: try:
await shutdown_event.wait() await shutdown_event.wait()
except (KeyboardInterrupt, asyncio.CancelledError): except (KeyboardInterrupt, asyncio.CancelledError):
@@ -246,7 +290,16 @@ async def main() -> None:
comm_stop_event.set() comm_stop_event.set()
if not client.is_closed(): if not client.is_closed():
await client.close() await client.close()
await asyncio.gather(bot_task, comm_task, return_exceptions=True) results = await asyncio.gather(bot_task, comm_task, return_exceptions=True)
for result in results:
if isinstance(result, BaseException) and not isinstance(
result, asyncio.CancelledError
):
# Already logged with full traceback inside the task; repeat
# the one-liner so it is the LAST thing in `docker logs`.
logger.critical("Task died: %r", result)
exit_code = 1
return exit_code
# *================================== Run # *================================== Run
@@ -264,4 +317,4 @@ if __name__ == "__main__":
) )
logger.critical(MSG) logger.critical(MSG)
sys.exit(MSG) sys.exit(MSG)
asyncio.run(main()) sys.exit(asyncio.run(main()))
+5 -2
View File
@@ -234,11 +234,14 @@ def comm_subroutine(stop_event: Optional[threading.Event] = None):
:param stop_event: optional :class:`threading.Event` shared with the caller :param stop_event: optional :class:`threading.Event` shared with the caller
to coordinate a cooperative shutdown. to coordinate a cooperative shutdown.
""" """
logger.setLevel(logging.DEBUG)
logger = logging.getLogger("discord") logger = logging.getLogger("discord")
logger.setLevel(logging.DEBUG)
logger.info("Started comms") logger.info("Started comms")
threads = [] threads = []
threads.append(threading.Thread(target=flask_debug)) # NOTE: flask_debug is the dev server bound to the SAME host:port as
# waitress - running both kills the comm layer with 'address in use'.
# Enable it only INSTEAD of waitress_run, never alongside.
# threads.append(threading.Thread(target=flask_debug))
threads.append(threading.Thread(target=waitress_run, daemon=True)) threads.append(threading.Thread(target=waitress_run, daemon=True))
threads.append( threads.append(
threading.Thread( threading.Thread(