Dockerization + ai review recomendations.

conjurer_librarian/conjurer_librarian.py

@@ -16,34 +16,52 @@ Functions:
 import asyncio
 import json
 import logging
-import netrc
+import os
 import threading
-import time
 from json.decoder import JSONDecodeError
 from logging import handlers
+from pathlib import Path
 from queue import Queue
+from typing import Dict, Optional
 
 import requests
 import scrape_bot
 import search_bot
-#import search_bot2 as search_bot
-from flask import Flask, jsonify, request
+# import search_bot2 as search_bot
+from flask import Flask, jsonify, request, abort
 from habanero import Crossref
 from waitress import serve
 
+try:
+    import netrc
+except ImportError:  # pragma: no cover
+    netrc = None
+
 # Constants
-NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
-HOST_ADDRESS = "192.168.1.192"
-PORT_ADDRESS = 5001
-MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
-SEND_RESULTS = "/conjurer"
-BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
 
-MAX_CR_RESULTS = 500
-#TEST PURPOSES ONLY!
-#MAX_CR_RESULTS = 5
 
-ENCODING = "utf-8"
+def _env(name: str, default: str) -> str:
+    return os.getenv(name, default)
+
+
+def _env_path(name: str, default: str) -> Path:
+    return Path(os.getenv(name, default)).expanduser().resolve()
+
+
+BASE_DIR = Path(
+    os.getenv("CONJURER_LIBRARIAN_BASE", str(Path(__file__).resolve().parent))
+)
+NETRC_FILE = _env_path("CONJURER_NETRC_FILE", str(Path.home() / ".netrc"))
+HOST_ADDRESS = _env("CONJURER_LIBRARIAN_HOST", "0.0.0.0")
+PORT_ADDRESS = int(_env("CONJURER_LIBRARIAN_PORT", "5001"))
+MAIN_BOT_ADDRESS = _env("CONJURER_MAIN_BOT", "http://127.0.0.1:5000")
+SEND_RESULTS = _env("CONJURER_LIBRARIAN_RESULTS_ENDPOINT", "/conjurer")
+MAX_CR_RESULTS = int(_env("CONJURER_LIBRARIAN_MAX_RESULTS", "500"))
+ENCODING = _env("CONJURER_ENCODING", "utf-8")
+API_KEY = os.getenv("CONJURER_API_KEY")
+LOGFILE_PATH = _env_path(
+    "CONJURER_LIBRARIAN_LOG", str(BASE_DIR / "librarian.log")
+)
 
 app = Flask(__name__)
 
@@ -51,6 +69,17 @@ librarian_queue = Queue()
 librarian_list = []
 
 
+def _service_headers() -> Dict[str, str]:
+    if API_KEY:
+        return {"X-Conjurer-Api-Key": API_KEY}
+    return {}
+
+
+def _authorize_request() -> None:
+    if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY:
+        abort(401)
+
+
 # trunk-ignore(pylint/R0902)
 class Librarian(object):
     """
@@ -81,11 +110,24 @@ class Librarian(object):
         - search_result_from_cr: A dictionary to store the search results from Crossref.
         - done: A flag indicating if the search is done.
         """
-        netrc_mod = netrc.netrc(NETRC_FILE)
-        auth_tokens = netrc_mod.authenticators("crossref")
+        mailto_contact: Optional[str] = os.getenv("CONJURER_CROSSREF_MAILTO")
+        if netrc:
+            try:
+                netrc_mod = netrc.netrc(str(NETRC_FILE))
+                auth_tokens = netrc_mod.authenticators("crossref")
+                if auth_tokens:
+                    mailto_contact = auth_tokens[0]
+            except (FileNotFoundError, netrc.NetrcParseError):
+                logging.getLogger("conjurer_librarian").warning(
+                    "Crossref credentials missing in netrc %s", NETRC_FILE
+                )
+        if not mailto_contact:
+            raise RuntimeError(
+                "Crossref credentials not configured. Set CONJURER_CROSSREF_MAILTO or add to netrc."
+            )
         self.cr = Crossref(
-            mailto=auth_tokens[0],
-            ua_string=f"Conjurer project. mailto:{auth_tokens[0]}"
+            mailto=mailto_contact,
+            ua_string=f"Conjurer project. mailto:{mailto_contact}"
         )
         self.query = query
         self.uuid = str(uuid)
@@ -132,7 +174,7 @@ class Librarian(object):
                 self.fetched = len(cr_result["message"]["items"])
                 self.app.logger.info(self.total)
                 self.app.logger.info(self.fetched)
-                time.sleep(0.1)
+                await asyncio.sleep(0.1)
 
         else:
             cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
@@ -411,18 +453,20 @@ class BackgroundTaskSearch(threading.Thread):
                 requests.post,
                 f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
                 json=result,
+                headers=_service_headers(),
                 timeout=360,
             )
             self.app.logger.info("SENT")
             result = await coroutine
             self.app.logger.info(result.status_code)
             self.app.logger.info("SEND CONFIRMED")
-            time.sleep(1)
+            await asyncio.sleep(1)
 
 
 # ==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
+    _authorize_request()
     """
     Endpoint for querying the database.
 
@@ -455,6 +499,7 @@ async def query_database():
 
 @app.route("/get_partial_result", methods=["POST"])
 async def get_partial():
+    _authorize_request()
     """
     Retrieves the partial result for a given UUID.
 
@@ -478,9 +523,10 @@ async def get_partial():
 # =======================================MAIN===================================================
 if __name__ == "__main__":
     app.logger.setLevel(logging.DEBUG)
+    LOGFILE_PATH.parent.mkdir(parents=True, exist_ok=True)
     h1 = handlers.RotatingFileHandler(
-        filename="D:\\logs\\librarian.log",
-        encoding="utf-8",
+        filename=str(LOGFILE_PATH),
+        encoding=ENCODING,
         mode="a",
         maxBytes=6 * 1024 * 1024,
         backupCount=6,
@@ -488,20 +534,24 @@ if __name__ == "__main__":
 
     app.logger.addHandler(h1)
     threads = []
-    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=waitress_run, daemon=True))
     # threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
+    bgtask.daemon = True
     threads.append(bgtask)
-    threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
+    threads.append(
+        threading.Thread(
+            target=scrape_bot.scraper, args=(app.logger,), daemon=True
+        )
+    )
     i = 0
-    for worker in threads:
-        try:
+    try:
+        for worker in threads:
             app.logger.info("App number: %s", i)
             i += 1
             worker.start()
-        except RuntimeError as e:
-            app.logger.error("Exploded")
-            print(str(e))
-    for worker in threads:
-        worker.join()
+        for worker in threads:
+            worker.join()
+    except KeyboardInterrupt:
+        app.logger.info("Shutdown requested - exiting librarian service")