diff --git a/bot.py b/bot.py index f5ec0a5..f0fa35c 100644 --- a/bot.py +++ b/bot.py @@ -706,6 +706,8 @@ async def get_file(ctx, source, link): "force_keyframes": True, }, ] + #TODO: Make sponsorblock work + sponsorblock_postprocessor = [] if platform == "win32": dir_path = "G:\\Muzyka\\Youtube" else: @@ -714,7 +716,6 @@ async def get_file(ctx, source, link): "proxy": "", "default_search": "ytsearch", "format": "bestaudio/best", - "postprocessors": sponsorblock_postprocessor, "noplaylist": True, "no_color": False, "paths": {"home": dir_path}, @@ -824,7 +825,7 @@ async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None): It is an optional parameter with a default value of 0, which means that if no value is provided for how_many, the function will return all search results, defaults to 0 (optional) """ - # TODO: Potem dorobić wyszukiwania po kawałkach słów + #TODO: Przerobić na webservice # i sieć neuronową z możliwością wyrażenia opinii o dopasowaniu. if slowa_kluczowe: word_list = slowa_kluczowe @@ -1299,7 +1300,8 @@ async def zagraj_mi_kawalek(ctx): async with ctx.typing(): search_time_glob = datetime.now() logger.info("Zaczynam szukać timestamp %s", datetime.now()) - file = await wyszukaj(ctx=ctx) + coro = asyncio.to_thread(wyszukaj,ctx=ctx) + file = await coro logger.info( "Koniec szukania(timestamp %s zajęło %s", datetime.now(), diff --git a/conjurer.service b/conjurer.service new file mode 100644 index 0000000..89b2c4a --- /dev/null +++ b/conjurer.service @@ -0,0 +1,11 @@ +[Unit] +Description=Conjurer +After=multi-user.target + +[Service] +Type=simple +ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/bot.py +Restart=on-abort + +[Install] +WantedBy=multi-user.target diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py new file mode 100644 index 0000000..e0c326c --- /dev/null +++ b/conjurer_librarian/conjurer_librarian.py @@ -0,0 +1,190 @@ +import re +import logging +import threading +import time +from flask import Flask, jsonify, request +import netrc +import json +from json.decoder import JSONDecodeError +from platform import uname +from sys import platform +from urllib.request import urlopen +from habanero import Crossref +from waitress import serve + +if platform in ("linux", "linux2"): + SEPARATOR_FILE_PATH = "/" + if "microsoft-standard" in uname().release: + NETRC_FILE = "/home/mtuszowski/.netrc" + else: + NETRC_FILE = "/home/pi/.netrc" +elif platform == "win32": + NETRC_FILE = "C:\\Users\\mtusz\\.netrc" + +#HOST_ADDRESS = "192.168.1.191" +HOST_ADDRESS = "0.0.0.0" +PORT_ADDRESS = 5001 + +class Librarian(object): + + def __init__(self) -> None: + netrc_mod = netrc.netrc(NETRC_FILE) + authTokens = netrc_mod.authenticators("crossref") + self.cr = Crossref(mailto=authTokens[0]) + self.limit = 1000 + self.fetched = 0 + self.hit = 0 + self.total = 0 + + def search_crossref(self, query, uuid): + search_result_from_cr = self.cr.works(query = query, limit = self.limit) + self.total = search_result_from_cr['message']['total-results'] + self.fetched = len(search_result_from_cr['message']['items']) + #add to sql with UUID + #sql row - title, doi, author, uuid, last retrieved, is_available + with open("results.json", "r+", encoding="UTF-8") as data_file: + # First we load existing data into a dict. + try: + file_data = json.load(data_file) + except JSONDecodeError: + file_data = [] + # Join new_data with file_data inside emp_details + summarized_results = [] + for item in search_result_from_cr['message']['items']: + container = "NO" + if "container-title" in item: + container = "YES" + summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) + result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], } + file_data.append(result) + print("Saving") + data_file.seek(0) + # convert back to json. + json.dump(file_data, data_file, indent=4) + return result + + + def check_if_exists(self,page_url): + #TODO: Spowolnic + #TODO: Sprawdzac po mirrorach + #TODO: sprawdzac w tym jebanym pliku + in_scihub_db = True + if not page_url.startswith(("http:", "https:")): + raise ValueError("URL must start with 'http:' or 'https:'") + # trunk-ignore(bandit/B310) + with urlopen(page_url) as response: + data = response.read() + text = data.decode("utf-8") + for line in text.splitlines(): + if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): + return None + if in_scihub_db: + return text + return None + + def download(self, page_url): + if response := self.check_if_exists(page_url): + for line in response.splitlines(): + if m := re.match(r".*