file share: route the bot's client through config + shared auth

file_search_functions was the only bot-side service client that hardcoded the
musician's LAN address and sent no auth header. Both musician endpoints it
calls (/get_share_list, /get_share_links) run _authorize_request(), so the
whole file-share feature 401'd on any deployment that set CONJURER_API_KEY -
which the deployment guides instruct you to do - and the hardcoded IP made a
musician on another host (the Proxmox/docker split) unreachable regardless.

Use FILE_SERVICE_ADDRESS + service_headers() like music_functions does, add
the two endpoint paths to constants alongside the existing ones, and set an
explicit timeout so a wedged musician can't hang the calling cog.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

conjurer_librarian/cut_pieces.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import argparse
+import logging
+import sys
+
+
+# =========================
+# Configuration constants
+# =========================
+
+MAX_CHUNK_SIZE_BYTES = 50 * 1024 * 1024  # 100 MB
+OUTPUT_SUFFIX = "_chunk.txt"
+
+
+# =========================
+# Logging
+# =========================
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_default_output_dir(input_file: Path) -> Path:
+    """
+    Return default output directory placed next to the input file.
+
+    Example:
+        /data/big.txt -> /data/big/
+    """
+    return input_file.with_suffix("")
+
+
+def write_chunk(output_dir: Path, chunk_index: int, lines: list[bytes], chunk_size: int) -> Path:
+    """
+    Write one chunk file and return its path.
+    """
+    output_file = output_dir / f"{chunk_index}{OUTPUT_SUFFIX}"
+
+    with output_file.open("wb") as file:
+        file.writelines(lines)
+
+    actual_size = output_file.stat().st_size
+
+    if actual_size != chunk_size:
+        logger.warning(
+            "Chunk size mismatch for %s: expected %d bytes, got %d bytes",
+            output_file,
+            chunk_size,
+            actual_size,
+        )
+
+    return output_file
+
+
+def split_txt_file(input_file: Path, output_dir: Path | None = None) -> None:
+    """
+    Split a text-like file into smaller files limited by MAX_CHUNK_SIZE_BYTES.
+
+    The file is processed in binary mode, so invalid UTF-8 or mixed encodings
+    do not break the split. Lines are never split.
+    """
+    if not input_file.exists():
+        raise FileNotFoundError(f"Input file does not exist: {input_file}")
+
+    if not input_file.is_file():
+        raise ValueError(f"Input path is not a file: {input_file}")
+
+    if MAX_CHUNK_SIZE_BYTES <= 0:
+        raise ValueError("MAX_CHUNK_SIZE_BYTES must be greater than zero")
+
+    if output_dir is None:
+        output_dir = get_default_output_dir(input_file)
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    logger.info("Input file: %s", input_file)
+    logger.info("Output directory: %s", output_dir)
+    logger.info("Max chunk size: %d bytes", MAX_CHUNK_SIZE_BYTES)
+
+    chunk_index = 0
+    current_lines: list[bytes] = []
+    current_size = 0
+    total_lines = 0
+    total_bytes = 0
+
+    with input_file.open("rb") as input_handle:
+        for line_number, line in enumerate(input_handle, start=1):
+            line_size = len(line)
+            total_lines = line_number
+            total_bytes += line_size
+
+            if line_size > MAX_CHUNK_SIZE_BYTES:
+                logger.warning(
+                    "Line %d is larger than max chunk size: %d bytes > %d bytes. "
+                    "It will be written as a separate chunk.",
+                    line_number,
+                    line_size,
+                    MAX_CHUNK_SIZE_BYTES,
+                )
+
+            if current_lines and current_size + line_size > MAX_CHUNK_SIZE_BYTES:
+                output_file = write_chunk(
+                    output_dir=output_dir,
+                    chunk_index=chunk_index,
+                    lines=current_lines,
+                    chunk_size=current_size,
+                )
+
+                logger.info(
+                    "Written chunk %d: %s (%d bytes)",
+                    chunk_index,
+                    output_file,
+                    current_size,
+                )
+
+                chunk_index += 1
+                current_lines = []
+                current_size = 0
+
+            current_lines.append(line)
+            current_size += line_size
+
+    if current_lines:
+        output_file = write_chunk(
+            output_dir=output_dir,
+            chunk_index=chunk_index,
+            lines=current_lines,
+            chunk_size=current_size,
+        )
+
+        logger.info(
+            "Written chunk %d: %s (%d bytes)",
+            chunk_index,
+            output_file,
+            current_size,
+        )
+
+        chunk_count = chunk_index + 1
+    else:
+        logger.info("Input file is empty. No chunks were created.")
+        chunk_count = 0
+
+    logger.info("Total lines processed: %d", total_lines)
+    logger.info("Total bytes processed: %d", total_bytes)
+    logger.info("Chunks created: %d", chunk_count)
+    logger.info("Done.")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Split a large TXT-like file into smaller line-safe chunks."
+    )
+
+    parser.add_argument(
+        "input_file",
+        type=Path,
+        help="Path to input TXT file.",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        type=Path,
+        default=None,
+        help=(
+            "Optional output directory. "
+            "By default, a directory named after the input file is created next to it."
+        ),
+    )
+
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+
+    try:
+        split_txt_file(args.input_file, args.output_dir)
+        return 0
+    except Exception:
+        logger.exception("Failed to split file")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

conjurer_librarian/scrape_bot.py

@@ -19,7 +19,7 @@ from requests import ConnectTimeout, Timeout
 import lib_paths
 
 SCR_DATABASE_PATH = os.getenv("CONJURER_LIBRARIAN_DB_PATH", r"C:\\Database\\chunks\\")
-SCR_FILENAME = os.getenv("CONJURER_LIBRARIAN_SCRAPE_CHUNK", "40_chunk.txt")
+SCR_FILENAME = os.getenv("CONJURER_LIBRARIAN_SCRAPE_CHUNK", "44_chunk.txt")
 SCR_ENCODING = os.getenv("CONJURER_ENCODING", "utf-8")
 
 WORK_Q = Queue()
@@ -40,7 +40,7 @@ def load_ndb_to_q(logger):
                         key = next(iter(ndb_database))
                         _ = ndb_database.pop(key)
                         logger.info(key)
-                        url = f"https://sci-hub.se/{key}"
+                        url = f"https://sci-hub.red/{key}"
                         WORK_Q.put([key, url, False])
                     except StopIteration:
                         break

constants.py

@@ -82,6 +82,8 @@ MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
 GET_MP3 = "/mp3"
 SEND_MP3 = "/update_mp3"
 GET_PLAYLIST = "/get_music"
+GET_SHARE_LIST = "/get_share_list"
+GET_SHARE_LINKS = "/get_share_links"
 ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
 CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
 

docker/compose.librarian.yaml

@@ -16,7 +16,7 @@ services:
     volumes:
       # Local DOI chunk database (0_chunk.txt ... N_chunk.txt). Read-only:
       # the search workers only read it (open mode "r").
-      - /srv/librarian/doi:/doi:ro
+      - /mnt/conjurer_swap/librarian_data/base_it1:/doi:ro
       # Runtime JSON state (cr_results/rr_results/not_in_db/s_results) -
       # seeded on first run, persisted here across restarts.
       - /srv/librarian/state:/lib_temp_files

file_search_functions.py

@@ -1,7 +1,29 @@
+"""Bot-side client for the musician's share-list / share-link endpoints.
+
+This module used to hardcode the musician's LAN address and send no auth
+header, which broke the file-share feature in two ways:
+
+* the musician's ``/get_share_list`` and ``/get_share_links`` both call
+  ``_authorize_request()``, so every request 401'd on any deployment that set
+  ``CONJURER_API_KEY`` — which the deployment guides tell you to do; and
+* the hardcoded IP ignored ``CONJURER_FILE_SERVICE``, so a musician running on
+  another host (the Proxmox/docker split) was unreachable.
+
+It now follows the same convention as the other service clients
+(``music_functions``): address from ``constants``, shared auth headers, and an
+explicit timeout so a wedged musician can't hang the bot.
+"""
 import requests
 
-# Base URL for Flask service
-BASE_URL = 'http://192.168.1.15:5000'
+from constants import (
+    FILE_SERVICE_ADDRESS,
+    GET_SHARE_LINKS,
+    GET_SHARE_LIST,
+    service_headers,
+)
+
+SERVICE_HEADERS = service_headers()
+REQUEST_TIMEOUT = 60
 
 
 def find_matches(entries, keywords):
@@ -12,7 +34,12 @@ def find_matches(entries, keywords):
     Returns: list of file paths
     """
     payload = {'entries': entries, 'keywords': keywords}
-    response = requests.post(f"{BASE_URL}/get_share_list", json=payload)
+    response = requests.post(
+        f"{FILE_SERVICE_ADDRESS}{GET_SHARE_LIST}",
+        json=payload,
+        headers=SERVICE_HEADERS,
+        timeout=REQUEST_TIMEOUT,
+    )
     response.raise_for_status()
     data = response.json()
     return data.get('files', [])
@@ -25,7 +52,12 @@ def publish(file_paths):
     Returns: list of published URLs
     """
     payload = {'file_paths': file_paths}
-    response = requests.post(f"{BASE_URL}/get_share_links", json=payload)
+    response = requests.post(
+        f"{FILE_SERVICE_ADDRESS}{GET_SHARE_LINKS}",
+        json=payload,
+        headers=SERVICE_HEADERS,
+        timeout=REQUEST_TIMEOUT,
+    )
     response.raise_for_status()
     data = response.json()
     return data.get('links', [])