From d9147115597dd9a5b6f69273084e3b27efb180b3 Mon Sep 17 00:00:00 2001 From: Polish Hammer Date: Thu, 30 Oct 2025 16:58:56 +0100 Subject: [PATCH] Tag: 1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intermediate commits (oldest → newest): - New files for flask services - Service - Minor fix 2 - Waitress - Linux conf files - Service - Bugfixing - Waitress - Install options - Work on deployment - Bugfixes - Work on deployment - name change and othre bugfixes - Installer - Test --- bot.py | 8 +- conjurer.service | 11 + conjurer_librarian/conjurer_librarian.py | 190 ++ .../install_librarian_service.sh | 11 + conjurer_librarian/requirements_librarian.txt | 3 + deploy.sh | 9 + file_webservice/conjurer_musician.py | 134 + file_webservice/conjurer_musician.service | 11 + file_webservice/install_musician_service.sh | 11 + file_webservice/requirements_musician.txt | 2 + install_main_bot.sh | 12 + mp3_list.service.py | 35 - requirements_bot.txt | 13 + results.json | 2742 +++++++++++++++++ spotify_dl/youtube.py | 4 +- sql_webservice/conjurer_librarian.py | 0 .../cut_large_file.py | 15 + .../import_to_mysql.py | 0 18 files changed, 3171 insertions(+), 40 deletions(-) create mode 100644 conjurer.service create mode 100644 conjurer_librarian/conjurer_librarian.py create mode 100644 conjurer_librarian/install_librarian_service.sh create mode 100644 conjurer_librarian/requirements_librarian.txt create mode 100755 deploy.sh create mode 100644 file_webservice/conjurer_musician.py create mode 100644 file_webservice/conjurer_musician.service create mode 100644 file_webservice/install_musician_service.sh create mode 100644 file_webservice/requirements_musician.txt create mode 100755 install_main_bot.sh delete mode 100644 mp3_list.service.py create mode 100644 requirements_bot.txt create mode 100644 results.json create mode 100644 sql_webservice/conjurer_librarian.py rename cut_scihub_database.py => utils/cut_large_file.py (72%) rename import_to_mysql.py => utils/import_to_mysql.py (100%) diff --git a/bot.py b/bot.py index f5ec0a5..f0fa35c 100644 --- a/bot.py +++ b/bot.py @@ -706,6 +706,8 @@ async def get_file(ctx, source, link): "force_keyframes": True, }, ] + #TODO: Make sponsorblock work + sponsorblock_postprocessor = [] if platform == "win32": dir_path = "G:\\Muzyka\\Youtube" else: @@ -714,7 +716,6 @@ async def get_file(ctx, source, link): "proxy": "", "default_search": "ytsearch", "format": "bestaudio/best", - "postprocessors": sponsorblock_postprocessor, "noplaylist": True, "no_color": False, "paths": {"home": dir_path}, @@ -824,7 +825,7 @@ async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None): It is an optional parameter with a default value of 0, which means that if no value is provided for how_many, the function will return all search results, defaults to 0 (optional) """ - # TODO: Potem dorobić wyszukiwania po kawałkach słów + #TODO: Przerobić na webservice # i sieć neuronową z możliwością wyrażenia opinii o dopasowaniu. if slowa_kluczowe: word_list = slowa_kluczowe @@ -1299,7 +1300,8 @@ async def zagraj_mi_kawalek(ctx): async with ctx.typing(): search_time_glob = datetime.now() logger.info("Zaczynam szukać timestamp %s", datetime.now()) - file = await wyszukaj(ctx=ctx) + coro = asyncio.to_thread(wyszukaj,ctx=ctx) + file = await coro logger.info( "Koniec szukania(timestamp %s zajęło %s", datetime.now(), diff --git a/conjurer.service b/conjurer.service new file mode 100644 index 0000000..89b2c4a --- /dev/null +++ b/conjurer.service @@ -0,0 +1,11 @@ +[Unit] +Description=Conjurer +After=multi-user.target + +[Service] +Type=simple +ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/bot.py +Restart=on-abort + +[Install] +WantedBy=multi-user.target diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py new file mode 100644 index 0000000..e0c326c --- /dev/null +++ b/conjurer_librarian/conjurer_librarian.py @@ -0,0 +1,190 @@ +import re +import logging +import threading +import time +from flask import Flask, jsonify, request +import netrc +import json +from json.decoder import JSONDecodeError +from platform import uname +from sys import platform +from urllib.request import urlopen +from habanero import Crossref +from waitress import serve + +if platform in ("linux", "linux2"): + SEPARATOR_FILE_PATH = "/" + if "microsoft-standard" in uname().release: + NETRC_FILE = "/home/mtuszowski/.netrc" + else: + NETRC_FILE = "/home/pi/.netrc" +elif platform == "win32": + NETRC_FILE = "C:\\Users\\mtusz\\.netrc" + +#HOST_ADDRESS = "192.168.1.191" +HOST_ADDRESS = "0.0.0.0" +PORT_ADDRESS = 5001 + +class Librarian(object): + + def __init__(self) -> None: + netrc_mod = netrc.netrc(NETRC_FILE) + authTokens = netrc_mod.authenticators("crossref") + self.cr = Crossref(mailto=authTokens[0]) + self.limit = 1000 + self.fetched = 0 + self.hit = 0 + self.total = 0 + + def search_crossref(self, query, uuid): + search_result_from_cr = self.cr.works(query = query, limit = self.limit) + self.total = search_result_from_cr['message']['total-results'] + self.fetched = len(search_result_from_cr['message']['items']) + #add to sql with UUID + #sql row - title, doi, author, uuid, last retrieved, is_available + with open("results.json", "r+", encoding="UTF-8") as data_file: + # First we load existing data into a dict. + try: + file_data = json.load(data_file) + except JSONDecodeError: + file_data = [] + # Join new_data with file_data inside emp_details + summarized_results = [] + for item in search_result_from_cr['message']['items']: + container = "NO" + if "container-title" in item: + container = "YES" + summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) + result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], } + file_data.append(result) + print("Saving") + data_file.seek(0) + # convert back to json. + json.dump(file_data, data_file, indent=4) + return result + + + def check_if_exists(self,page_url): + #TODO: Spowolnic + #TODO: Sprawdzac po mirrorach + #TODO: sprawdzac w tym jebanym pliku + in_scihub_db = True + if not page_url.startswith(("http:", "https:")): + raise ValueError("URL must start with 'http:' or 'https:'") + # trunk-ignore(bandit/B310) + with urlopen(page_url) as response: + data = response.read() + text = data.decode("utf-8") + for line in text.splitlines(): + if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): + return None + if in_scihub_db: + return text + return None + + def download(self, page_url): + if response := self.check_if_exists(page_url): + for line in response.splitlines(): + if m := re.match(r".*