From 6c9f01971a5f5c5fe4c350853822baa5364a8aae Mon Sep 17 00:00:00 2001 From: Migatu Date: Fri, 29 Mar 2024 20:19:21 +0100 Subject: [PATCH] Bugfixes asdasdasd :) asa asdasda aa :) asa aa :) :) sa as aa aa aa aaa as 11 aa 11 xxx dd ds aa a1 aa Test Suppress Move Installer aaa Serwis `12` aa xd asasa xd aa asa aaa :) Hejka a xd aa aaas 555 aaa asa aa 11 aaa aaa as sa async asda 00 123 123 aa as ss as a sa aa 1 RES T Hejko aa 11 AA test Test --- conjurer_librarian/conjurer_librarian.py | 167 + .../install_librarian_service.sh | 11 + conjurer_librarian/requirements_librarian.txt | 3 + deploy.sh | 2 +- file_webservice/conjurer_musician.py | 90 +- file_webservice/conjurer_musician.service | 11 + file_webservice/install_musician_service.sh | 11 + file_webservice/requirements_musician.txt | 2 + install.sh => install_main_bot.sh | 7 +- requirements.txt => requirements_bot.txt | 3 +- results.json | 2742 +++++++++++++++++ spotify_dl/youtube.py | 4 +- .../cut_large_file.py | 15 + .../import_to_mysql.py | 0 14 files changed, 3047 insertions(+), 21 deletions(-) create mode 100644 conjurer_librarian/conjurer_librarian.py create mode 100644 conjurer_librarian/install_librarian_service.sh create mode 100644 conjurer_librarian/requirements_librarian.txt create mode 100644 file_webservice/conjurer_musician.service create mode 100644 file_webservice/install_musician_service.sh create mode 100644 file_webservice/requirements_musician.txt rename install.sh => install_main_bot.sh (57%) rename requirements.txt => requirements_bot.txt (85%) create mode 100644 results.json rename cut_scihub_database.py => utils/cut_large_file.py (72%) rename import_to_mysql.py => utils/import_to_mysql.py (100%) diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py new file mode 100644 index 0000000..b0942aa --- /dev/null +++ b/conjurer_librarian/conjurer_librarian.py @@ -0,0 +1,167 @@ +import re +import logging +import threading +from flask import Flask, jsonify, request +import netrc +import json +from json.decoder import JSONDecodeError +from platform import uname +from sys import platform +from urllib.request import urlopen +from habanero import Crossref +from waitress import serve + +if platform in ("linux", "linux2"): + SEPARATOR_FILE_PATH = "/" + if "microsoft-standard" in uname().release: + NETRC_FILE = "/home/mtuszowski/.netrc" + else: + NETRC_FILE = "/home/pi/.netrc" +elif platform == "win32": + NETRC_FILE = "C:\\Users\\mtusz\\.netrc" + +#HOST_ADDRESS = "192.168.1.191" +HOST_ADDRESS = "0.0.0.0" +PORT_ADDRESS = 5001 + +class Librarian(object): + + def __init__(self) -> None: + netrc_mod = netrc.netrc(NETRC_FILE) + authTokens = netrc_mod.authenticators("crossref") + self.cr = Crossref(mailto=authTokens[0]) + self.last_search = "" + self.limit = 20 + + def search_crossref(self, query, uuid): + self.last_search = self.cr.works(query = query, limit = self.limit) + #add to sql with UUID + #sql row - title, doi, author, uuid, last retrieved, is_available + with open("results.json", "r+", encoding="UTF-8") as data_file: + # First we load existing data into a dict. + try: + file_data = json.load(data_file) + except JSONDecodeError: + file_data = [] + # Join new_data with file_data inside emp_details + summarized_results = [] + for item in self.last_search['message']['items']: + container = "NO" + if "container-title" in item: + container = "YES" + print(item['title']) + print(item['DOI']) + + summarized_results.append({"DOI" : item['DOI'], "title": item['title'], "type": item['type'], "container": container}) + result = {"UUID": uuid, "total_results": self.last_search['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.last_search['message']['items'], } + file_data.append(result) + print("Saving") + data_file.seek(0) + # convert back to json. + json.dump(file_data, data_file, indent=4) + + + def check_if_exists(self,page_url): + in_scihub_db = True + if not page_url.startswith(("http:", "https:")): + raise ValueError("URL must start with 'http:' or 'https:'") + # trunk-ignore(bandit/B310) + with urlopen(page_url) as response: + info = response.info() + print(info.get_content_type()) + data = response.read() + text = data.decode("utf-8") + for line in text.splitlines(): + if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): + return None + if in_scihub_db: + return text + return None + + def download(self, page_url): + if response := self.check_if_exists(page_url): + for line in response.splitlines(): + if m := re.match(r".*