diff --git a/latex_functions.py b/latex_functions.py index a322e36..295b0eb 100644 --- a/latex_functions.py +++ b/latex_functions.py @@ -3,11 +3,10 @@ from __future__ import annotations import asyncio +import logging import re -import shlex import tempfile import zipfile -import logging from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -29,25 +28,67 @@ def is_tex_attachment(att, max_mb: int) -> bool: ) -async def run_latex_two_passes( - tex_filename: str, workdir: Path, tex_engine: str, timeout_sec: int, logger:str=None -) -> Tuple[bool, str, Path]: - cmd = f"{tex_engine} -interaction=nonstopmode -halt-on-error -file-line-error -no-shell-escape {shlex.quote(tex_filename)}" - logger = logging.getLogger(logger) if logger else None - if logger: - logger.debug("LaTeX cmd: %s (cwd=%s)", cmd, workdir) +def _build_cmd(engine: str, tex_filename: str) -> list[str]: + base = (engine or "").strip().lower() + if base in ("pdflatex", "xelatex", "lualatex"): + return [ + base, + "-interaction=nonstopmode", + "-halt-on-error", + "-file-line-error", + "-no-shell-escape", + tex_filename, + ] + if base == "latexmk": + return [ + "latexmk", + "-pdf", + "-interaction=nonstopmode", + "-halt-on-error", + "-file-line-error", + tex_filename, + ] + if base == "tectonic": + return [ + "tectonic", + "-X", + "compile", + "--keep-logs", + "--outdir", + ".", + tex_filename, + ] + return [ + "pdflatex", + "-interaction=nonstopmode", + "-halt-on-error", + "-file-line-error", + "-no-shell-escape", + tex_filename, + ] - async def _run_once() -> Tuple[int, str]: - proc = await asyncio.create_subprocess_shell( - cmd, + +async def run_latex_two_passes( + tex_filename: str, + workdir: Path, + tex_engine: str, + timeout_sec: int, + logger: str = None, +): + logger = logging.getLogger(logger) if logger else None + + async def run_once(cmd: list[str]) -> tuple[int, str]: + proc = await asyncio.create_subprocess_exec( + *cmd, cwd=str(workdir), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, ) try: out_b, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout_sec) - out_t = (out_b or b"").decode("utf-8", errors="replace") - return proc.returncode or 0, out_t + return (proc.returncode or 0), (out_b or b"").decode( + "utf-8", errors="replace" + ) except asyncio.TimeoutError: try: proc.kill() @@ -55,27 +96,39 @@ async def run_latex_two_passes( raise return 124, f"[Timeout] Compilation exceeded {timeout_sec}s.\n" - logs: List[str] = [] - rc1, out1 = await _run_once() + logs = [] + cmd = _build_cmd(tex_engine, tex_filename) + if logger: + logger.debug("LaTeX cmd: %s (cwd=%s)", " ".join(cmd), workdir) + + rc1, out1 = await run_once(cmd) logs.append(out1) + second_pass = not ((tex_engine or "").strip().lower() == "tectonic") + if rc1 != 0: log_text = "".join(logs) - if logger: - logger.debug("LaTeX pass1 FAIL %s\n%s", tex_filename, log_text[-2000:]) - return False, log_text, workdir / (Path(tex_filename).stem + ".pdf") + pdf = workdir / (Path(tex_filename).stem + ".pdf") + return False, log_text, pdf + + if second_pass: + rc2, out2 = await run_once(cmd) + logs.append(out2) + ok = rc2 == 0 + else: + ok = rc1 == 0 - rc2, out2 = await _run_once() - logs.append(out2) log_text = "".join(logs) pdf = workdir / (Path(tex_filename).stem + ".pdf") - ok = (rc2 == 0) and pdf.exists() - if logger: - logger.debug("LaTeX pass2 %s ok=%s\n%s", tex_filename, ok, log_text[-2000:]) + ok = ok and pdf.exists() return ok, log_text, pdf async def compile_single_tex_bytes( - tex_bytes: bytes, filename: str, tex_engine: str, timeout_sec: int, logger:str=None + tex_bytes: bytes, + filename: str, + tex_engine: str, + timeout_sec: int, + logger: str = None, ) -> Dict[str, Optional[object]]: if not is_safe_tex_name(filename): return { @@ -107,7 +160,7 @@ async def compile_single_tex_bytes( async def compile_zip_to_zip( - zip_bytes: bytes, tex_engine: str, timeout_sec: int, logger:str=None + zip_bytes: bytes, tex_engine: str, timeout_sec: int, logger: str = None ) -> Tuple[bytes, List[str]]: failed: List[str] = [] out_pdf_paths: List[Path] = [] @@ -143,6 +196,7 @@ async def compile_zip_to_zip( failed.append(f"{tex.name} (exception: {e})") if logger: logger.debug("BATCH EXC %s: %s", tex.name, e) + raise out_zip = td_path / "compiled_pdfs.zip" with zipfile.ZipFile(out_zip, "w", compression=zipfile.ZIP_DEFLATED) as zf: @@ -153,7 +207,7 @@ async def compile_zip_to_zip( async def ask_openai_diagnosis( - log_text: str, tex_excerpt: str, model: str, logger:str = None + log_text: str, tex_excerpt: str, model: str, logger: str = None ) -> str: logger = logging.getLogger(logger) if logger else None sys = ( diff --git a/librarian_commands.py b/librarian_commands.py index 0fe8d7f..3368be6 100644 --- a/librarian_commands.py +++ b/librarian_commands.py @@ -8,6 +8,7 @@ from queue import Empty import discord import pdf2image +import fitz import PyPDF2 import requests from discord.ext import commands, tasks @@ -53,20 +54,33 @@ class DataModule(commands.Cog): filename = res[random.randrange(0, len(res) - 1)] # select random page file = open(DIR_PATH_SADOX + filename, "rb") - readpdf = PyPDF2.PdfReader(file) - totalpages = len(readpdf.pages) - # trunk-ignore(bandit/B311) - page = random.randrange(1, totalpages) - # convert page to image - image = pdf2image.convert_from_path( - DIR_PATH_SADOX + filename, first_page=page, last_page=page - ) - byte_io_stream = io.BytesIO() - image[0].save(byte_io_stream, "JPEG") - byte_io_stream.seek(0) - byte_io_stream.name = "image.jpg" - file = discord.File(byte_io_stream) - await ctx.send(file=file) + if True: + doc = fitz.open(DIR_PATH_SADOX + filename) + totalpages = len(doc) + page_index = random.randrange(0, totalpages) + page = doc.load_page(page_index) + mat = fitz.Matrix(2.0, 2.0) # powiększenie + pix = page.get_pixmap(matrix=mat, alpha=False) + + byte_io_stream = io.BytesIO(pix.tobytes("png")) + byte_io_stream.seek(0) + byte_io_stream.name = "image.png" + await ctx.send(file=discord.File(byte_io_stream)) + else: #legacy + readpdf = PyPDF2.PdfReader(file) + totalpages = len(readpdf.pages) + # trunk-ignore(bandit/B311) + page = random.randrange(1, totalpages) + # convert page to image + image = pdf2image.convert_from_path( + DIR_PATH_SADOX + filename, first_page=page, last_page=page + ) + byte_io_stream = io.BytesIO() + image[0].save(byte_io_stream, "JPEG") + byte_io_stream.seek(0) + byte_io_stream.name = "image.jpg" + file = discord.File(byte_io_stream) + await ctx.send(file=file) self.logger.info("Get sadox completed") @tasks.loop(seconds=3) diff --git a/requirements_bot.txt b/requirements_bot.txt index 9eeca53..14d162d 100644 --- a/requirements_bot.txt +++ b/requirements_bot.txt @@ -13,6 +13,7 @@ spotipy tiktoken PyNaCl flask[async] +fitz waitress clickupython assemblyai[extras]