gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tag: 1.3

Browse Source 
Intermediate commits (oldest → newest):
- Almost deployment versio - for full deployment change parameteres in search_bot
- Preparation for deployment
- Main bot (#1)
- Search webservice (#2)
- Music service (#3)
- FIx
This commit is contained in:
Polish Hammer
2025-10-30 16:58:58 +01:00
parent 2988bd8205
commit 04638c9141
15 changed files with 463 additions and 161870 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot.py
+200 -141
View File
@@ -46,7 +46,12 @@ from spotify_dl import youtube as youtube_download
from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q
# *=========================== Classes definition
# The above code is defining a TypedDict called `Music_Config` in Python. This TypedDict has three
# keys: "ctx" which is an optional string, "queue" which is a list of strings, and "requester" which
# is a list of strings. This TypedDict is used to define the structure of a dictionary that should
# have these specific keys and value types.
Music_Config = TypedDict(
"Music_Config",
{
@@ -58,6 +63,10 @@ Music_Config = TypedDict(
class QueryControl:
"""
This class `QueryControl` is used to manage queries with information about the author, UUID,
content, logger, context, and replies.
"""
def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
self.author = query_author
self.uuid = query_uuid
@@ -70,6 +79,95 @@ class QueryControl:
)
self.replies = []
class MusicFileList(object):
"""
The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a
file service or local directory and update the list with new items.
"""
def __init__(self, uplogger) -> None:
self.music_file_list = []
self.file_service_active = False
self.logger = uplogger
self.logger.info("Created Playlist organizer class")
def refresh_file_list(self):
"""
The `refresh_file_list` function attempts to connect to a file service to retrieve a list of music
files, and if unsuccessful, it populates the list by scanning a local directory for .mp3 files.
"""
try:
self.logger.info("Attempt to connect to file service")
response = requests.get(f"{FILE_SERVICE_ADDRESS}{GET_MP3}", timeout=360)
self.music_file_list = response.json()["music_file_list"]
self.file_service_active = True
except requests.exceptions.RequestException as e:
for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
temp_music_file = mp3_item.as_posix()
if platform == "win32":
temp_music_file = temp_music_file.replace("/", "\\")
self.music_file_list.append(temp_music_file)
self.file_service_active = False
self.logger.error(e.strerror)
self.logger.error("Service Unavailable")
def get_file_list(self):
"""
The `get_file_list` function returns the list of music files stored in the object.
:return: The `music_file_list` attribute is being returned.
"""
return self.music_file_list
def update_file_list(self, item):
"""
The `update_file_list` function appends an item to a music file list and sends a POST request with
the item data to a file service address.
:param item: The `item` parameter in the `update_file_list` method is the file that you want to add
to the `music_file_list` attribute of the class instance. It is then sent as a JSON payload in a
POST request to a file service address along with the endpoint `SEND_MP3`
"""
self.music_file_list.append(item)
post_data = {"item": str(item)}
requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
class DoSearchView(discord.ui.View):
def __init__(self, _uuid, result_list, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.uuid = _uuid
self.result_list = result_list
@discord.ui.button(
label="Nastepna strona", style=discord.ButtonStyle.primary, row=0
)
async def next_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
@discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0)
async def reset_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
@discord.ui.button(
label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0
)
async def perv_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
# *=========================================== Predefines
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -93,6 +191,8 @@ FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
GET_MP3 = "/mp3"
SEND_MP3 = "/update_mp3"
GET_PLAYLIST = "/get_music"
ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
CLEAR_PRIO = "/clear_pr_pls"
LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
SEND_QUERY = "/query"
TIME_BETWEEN_CALLS = 100000
@@ -193,60 +293,6 @@ logger.info("Playlist generation")
# *=========================================== Load Data files
class MusicFileList(object):
"""
The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a
file service or local directory and update the list with new items.
"""
def __init__(self, uplogger) -> None:
self.music_file_list = []
self.file_service_active = False
self.logger = uplogger
self.logger.info("Created Playlist organizer class")
def refresh_file_list(self):
"""
The `refresh_file_list` function attempts to connect to a file service to retrieve a list of music
files, and if unsuccessful, it populates the list by scanning a local directory for .mp3 files.
"""
try:
self.logger.info("Attempt to connect to file service")
response = requests.get(f"{FILE_SERVICE_ADDRESS}{GET_MP3}", timeout=360)
self.music_file_list = response.json()["music_file_list"]
self.file_service_active = True
except requests.exceptions.RequestException as e:
for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
temp_music_file = mp3_item.as_posix()
if platform == "win32":
temp_music_file = temp_music_file.replace("/", "\\")
self.music_file_list.append(temp_music_file)
self.file_service_active = False
self.logger.error(e.strerror)
self.logger.error("Service Unavailable")
def get_file_list(self):
"""
The `get_file_list` function returns the list of music files stored in the object.
:return: The `music_file_list` attribute is being returned.
"""
return self.music_file_list
def update_file_list(self, item):
"""
The `update_file_list` function appends an item to a music file list and sends a POST request with
the item data to a file service address.
:param item: The `item` parameter in the `update_file_list` method is the file that you want to add
to the `music_file_list` attribute of the class instance. It is then sent as a JSON payload in a
POST request to a file service address along with the endpoint `SEND_MP3`
"""
self.music_file_list.append(item)
post_data = {"item": str(item)}
requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
music_file_list = []
music_file_list = MusicFileList(logger)
music_file_list.refresh_file_list()
@@ -283,6 +329,8 @@ for key in word_reactions:
word_reactions[key][2] = datetime.now()
logger.info("Done")
# *=========================================== Create Client
logger.info("Creating discord bot")
client = commands.Bot(intents=intents, command_prefix="$")
@@ -342,8 +390,8 @@ async def on_message(message):
if isinstance(message.channel, discord.DMChannel):
channel = client.get_channel(1064888712565100614)
await channel.send("Słyszałem ja żem że: " + message.content)
else:
channel = message.channel
return
channel = message.channel
await client.process_commands(message)
message.content = message.content.lower()
@@ -682,10 +730,40 @@ async def check_music():
@tasks.loop(seconds=3)
# trunk-ignore(pylint/R0915)
async def check_data_q():
channel = client.get_channel(1062047367337095268)
try:
data = IN_COMM_Q.get(block=False)
await channel.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
entries = []
if data.stop:
searcher = data.author
ctx = data.ctx
query = data.content
l_p = 1
for doi in data.entries:
logger.info(doi)
desc = data.entries[doi]
title = desc['Title'][0]
entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n")
l_p+=1
message = '*Z podłogi wysuwa się winda na książki*'
message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query}'
message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
if len(entries)<1:
message += "niestety nie ma nic"
elif len(entries)>=1 and len(entries)<5:
message += " znajduje się coś:\n"
for item in entries:
message+=item
else:
message+= " znajduje się cholernie dużo:\n"
for item in entries:
message+=item
if len(message) > 1500:
await ctx.send(message)
message = ""
await ctx.send(message)
#Kept for sentimental reasons
#await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
except Empty:
pass
@@ -1047,8 +1125,10 @@ async def get_file(ctx, source, link):
dir_path = None
item_type, item_id = spotify.parse_spotify_url(link)
if item_id:
file_list = spotify.fetch_tracks(spotify_ctrl, item_type, link)
directory_name = spotify.get_item_name(spotify_ctrl, item_type, item_id)
coro = asyncio.to_thread(spotify.fetch_tracks, args=(spotify_ctrl, item_type, link))
file_list = await coro
coro = asyncio.to_thread(spotify.get_item_name, args=(spotify_ctrl, item_type, item_id))
directory_name = await coro
url_data = {"urls": []}
url_dict = {}
url_dict["save_path"] = Path(
@@ -1275,6 +1355,7 @@ async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None):
json=jrequest,
timeout=360,
)
return_data = await coroutine
if not return_data.status_code == 200:
await ctx.send("Wołaj szefa - coś się wyjebało")
@@ -1700,6 +1781,63 @@ async def zagraj_mi_kawalek(ctx):
"Obawiam się że nie mogę nic znaleźć u siebie. Spróbuj komendy '$dej_co_ze_spotifaja' jeśli masz link"
)
@client.hybrid_command(
name="dodaj_do_ulubionych",
description="Dodaje do listy ulubionych w radiu",
guild=discord.Object(id=664789470779932693),
)
async def dodaj_do_ulubionych(ctx):
"""
Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
Rest of the line are search terms.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. This parameter is required in all
Discord.py commands
"""
async with ctx.typing():
logger.info("Zaczynam szukać timestamp %s", datetime.now())
dlugosc_playlisty = ctx.message.content.split()[1]
word_list = ctx.message.content.split()
jrequest = {
"lista_slow": word_list,
"dlugosc_plejlisty": dlugosc_playlisty,
"UUID": str(uuid.uuid4()),
}
coroutine = asyncio.to_thread(
requests.post,
f"{FILE_SERVICE_ADDRESS}{ADD_TO_PRIO_PLAYLIST}",
json=jrequest,
timeout=360,
)
_ = await coroutine
@client.hybrid_command(
name="wyczysc_ulubione",
description="Czysci liste ulubionych w radiu",
guild=discord.Object(id=664789470779932693),
)
async def wyczysc_ulubione(ctx):
"""
Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
Rest of the line are search terms.
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
represents the context in which the command was invoked, including information such as the message,
the channel, the server, and the user who invoked the command. This parameter is required in all
Discord.py commands
"""
async with ctx.typing():
coroutine = asyncio.to_thread(
requests.post,
f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
json={},
timeout=360,
)
_ = await coroutine
@client.hybrid_command(
name="zrob_mi_plejliste",
@@ -2065,44 +2203,6 @@ async def parametry_muzyki_mego_ludu(
logger.info(exce)
class DoSearchView(discord.ui.View):
def __init__(self, uuid, result_list, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.uuid = uuid
self.result_list = result_list
@discord.ui.button(
label="Nastepna strona", style=discord.ButtonStyle.primary, row=0
)
async def next_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
@discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0)
async def reset_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
@discord.ui.button(
label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0
)
async def perv_callback(self, interaction, button):
# emoji = await ctx.guild.fetch_emoji(id)
# button.emoji=1102174413887111189
logger.info("Pressed")
await interaction.response.edit_message(
content="You clicked the button!"
) # Send a message when the button is clicked
@client.hybrid_command(
name="wyszukaj_linki_do_dokumentow",
description="Szuka linkow doi w bazie crossref i podaje linki do scihuba",
@@ -2111,8 +2211,8 @@ class DoSearchView(discord.ui.View):
async def wyszukaj_linki_do_dokumentow(ctx):
query = ctx.message.content
query_uuid = uuid.uuid4()
# query_uuid = "test"
# 3 wyslij Librariana
#TODO: TESTING ONLY!!
#query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
json_query = {"UUID": str(query_uuid), "query": str(query)[30:], "page": 1}
coroutine = asyncio.to_thread(
requests.post,
@@ -2145,46 +2245,6 @@ async def wyszukaj_linki_do_dokumentow(ctx):
await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+
" Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.")
'''
async def wyswietl_wyniki_wyszukiwania(ctx):
# 4 zwroc 4 wyniki od librariana
# 5 przyciski przy każdym wierszu pozwalają sprawdzić czy plik istnieje na scihubie - jeśli tak podaje link do sciagniecia
# 6 ostatni wiersz to dalej, wróć, skasuj, zaciagnij kolejne wyniki (jeśli jest ich więcej niż 1000)
result_view = DoSearchView(
uuid=str(query_uuid), result_list=query_response.json()["data"]
)
resulttab = []
resultlink = []
hit, fetched, total, _ = query_response.json()["data"][0]
answer = query_response.json()["data"][1]
for item in answer:
resulttab.append(str(item["title"]))
item_link = "https://sci-hub.se/" + str(item["DOI"])
resultlink.append(item_link)
logger.info(len(resulttab))
for i in range(0, min(4, len(resulttab))):
logger.info(i)
logger.info(resulttab[i])
logger.info(resultlink[i])
url = resultlink[i]
logger.info(url)
temp_label = str(resulttab[i])
temp_label = temp_label.strip("[']")
result_view.add_item(
discord.ui.Button(label=str(temp_label[0:79]), row=i + 1, url=url)
)
labelka = (
"Wyniki wyszukiwania dla: "
+ str(query)[30:]
+ f"\nWszystkich {total} sprawdzonych: {fetched} znalezionych: {hit} "
) # ile znaleziono artykułów w ogóle, ile zostało po refine
await ctx.send(
labelka, view=result_view
) # Send a message with our View class that contains the button
'''
@client.hybrid_command(
name="krecimy_pornola",
@@ -2223,7 +2283,6 @@ async def krecimy_pornola(ctx):
# *================================== Run
if __name__ == "__main__":
# TODO: wpakowac kondzia w wątki odpalic asynchronicznie rownolegle z flaskiem do sterowania nim
logger.info("Starting discord bot")
threads = []
logger.info("Starting discord bot: Creating threads")
bot_classes.py
View File
bot_commands.py
View File
bot_functions.py
View File
communication_subroutine.py
+61 -15
View File
@@ -1,6 +1,7 @@
import logging
import threading
import json
import time
from queue import Queue, Empty
from flask import Flask, jsonify, request
from waitress import serve
@@ -9,28 +10,34 @@ HOST_ADDRESS = "192.168.1.191"
PORT_ADDRESS = 5000
OUT_COMM_Q = Queue()
IN_COMM_Q = Queue()
internal_q = Queue()
awaiting_q = []
incoming_q = Queue()
app = Flask(__name__)
@app.route("/conjurer", methods=["POST"])
def answer_external_command():
"""
The function `answer_external_command` logs the request data, loads the data as JSON, logs the
record, and then puts the record into an incoming queue before returning a success message.
:return: The function `answer_external_command()` is returning a JSON response with the message
"SUCCESS".
"""
app.logger.info(request)
record = json.loads(request.data)
app.logger.info(record)
app.logger.info("DATA RECEIVED")
while True:
try:
q_data = internal_q.get(block=False)
except Empty:
break
if q_data.uuid in record:
pass
incoming_q.put(record)
return jsonify("SUCCESS")
@app.route("/conjurer", methods=["GET"])
def check_alive():
"""
The function "check_alive" returns a JSON response with the message "ALIVE" when the "/conjurer"
route is accessed via a GET request.
:return: The code snippet is a Flask route that listens for GET requests to the "/conjurer"
endpoint. When a GET request is made to this endpoint, the function check_alive() is called, which
returns a JSON response with the message "ALIVE".
"""
return jsonify("ALIVE")
def flask_debug(_logger):
@@ -52,19 +59,58 @@ def waitress_run(_logger):
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
def scan_queue(_logger):
"""
The function `scan_queue` reads data from a queue, logs it, and appends it to another queue.
:param _logger: The `_logger` parameter is typically an instance of a logging object that is used to
record and store log messages. It is commonly used to track the flow of the program, record errors,
and provide information for debugging purposes. In this code snippet, the `_logger` object is used
to log the
"""
while True:
data = OUT_COMM_Q.get()
_logger.info(data)
internal_q.put(data)
query_list.append(data)
awaiting_q.append(data)
def scan_incoming(_logger):
"""
The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data
is found.
:param _logger: The `_logger` parameter in the `scan_incoming` function is a logger object that is
used to log messages or information during the execution of the function. It is typically used for
debugging, monitoring, or tracking the flow of the program. In this case, the `_logger` is being
used
"""
while True:
try:
answer = incoming_q.get(block=False)
_logger.info("DATA FOUND")
for record in awaiting_q:
if record.uuid in answer.keys():
record.stop = True
record.entries = answer[record.uuid]
IN_COMM_Q.put(record)
except Empty:
time.sleep(1)
def comm_subroutine(logger):
"""
The `comm_subroutine` function starts multiple threads to run different tasks concurrently.
:param logger: The `logger` parameter in the `comm_subroutine` function is an instance of a logger
object that is used for logging messages at various levels (e.g., debug, info, warning, error). In
the provided code snippet, the logger is used to log messages at the "info" level
"""
#logger.setLevel(logging.DEBUG)
logger.info("Started")
threads = []
threads.append(threading.Thread(target=flask_debug, args=(logger,)))
#threads.append(threading.Thread(target=waitress_run, args=(logger,)))
#threads.append(threading.Thread(target=flask_debug, args=(logger,)))
threads.append(threading.Thread(target=waitress_run, args=(logger,)))
threads.append(threading.Thread(target=scan_queue,args=(logger,)))
threads.append(threading.Thread(target=scan_incoming,args=(logger,)))
for worker in threads:
worker.start()
conjurer_librarian/conjurer_librarian.py
+53 -23
View File
@@ -12,9 +12,8 @@ from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
import scrape_bot
import requests
from typing import Callable
from uuid import UUID
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
HOST_ADDRESS = "192.168.1.192"
@@ -22,7 +21,7 @@ PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
MAX_CR_RESULTS = 150000
ENCODING = "utf-8"
app = Flask(__name__)
@@ -37,13 +36,15 @@ class Librarian(object):
self.cr = Crossref(mailto=authTokens[0])
self.query = query
self.uuid = str(uuid)
self.limit = 3000
self.limit = MAX_CR_RESULTS
self.fetched = 0
self.hit = 0
self.total = 0
self.app = app
self.logger = app.logger
self.live_results = []
self.final_result = []
self.not_in_db = []
self.search_result_from_cr = {}
self.done = False
@@ -62,8 +63,7 @@ class Librarian(object):
self.fetched += len(result['message']['items'])
self.logger.info (self.total)
self.logger.info(self.fetched)
time.sleep(1)
time.sleep(0.1)
self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
@@ -142,20 +142,20 @@ class Librarian(object):
temp.append([item["DOI"], item])
refined_result = await self.check_if_exists(temp)
temp = []
for item in refined_result:
if item["exists"]:
temp.append(item["data"])
refined_result = temp
self.hit = len(refined_result)
return refined_result
self.final_result.append(item["data"])
else:
self.not_in_db.append(item["data"])
self.hit = len(self.final_result)
return self.final_result
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
async def answer_query(self):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
self.logger.info(f"Search started {self.uuid}")
uuid_found = False
database = None
query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
@@ -165,11 +165,19 @@ class Librarian(object):
database = json.load(cr_file)
except JSONDecodeError:
pass
#TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
if database:
for item in database.keys():
if item in self.uuid:
self.result = await self.refine_query(database)
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
answer = await self.search_crossref(query=query)
database.update(answer)
@@ -177,7 +185,6 @@ class Librarian(object):
cr_file.seek(0)
json.dump(database,cr_file)
self.result = await self.refine_query(answer)
return self.result
else:
answer = await self.search_crossref(query=query)
database = answer
@@ -187,7 +194,15 @@ class Librarian(object):
self.result = await self.refine_query(answer)
self.app.logger.info("Refined")
self.done = True
return self.result
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
#============================= FLASK INTERNALS===============================
def flask_debug():
@@ -214,18 +229,34 @@ class BackgroundTaskSearch(threading.Thread):
async def _run(self):
while True:
database = None
ndb_database = None
librarian = librarian_queue.get()
self.app.logger.info("STARTED")
result = await librarian.answer_query()
result = {librarian.uuid: result}
self.app.logger.info("Saving to file")
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
except JSONDecodeError:
pass
if ndb_database:
ndb_database.update(librarian.not_in_db)
else:
ndb_database = librarian.not_in_db
ndb_file.truncate(0)
ndb_file.seek(0)
json.dump(ndb_database, ndb_file)
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
except JSONDecodeError:
pass
if database:
database.extend(result)
self.app.logger.info(database)
self.app.logger.info(result)
database.update(result)
else:
database = result
self.app.logger.info("DUMPING DATA")
@@ -242,7 +273,8 @@ class BackgroundTaskSearch(threading.Thread):
timeout=360,
)
self.app.logger.info("SENT")
_ = await coroutine
result = await coroutine
self.app.logger.info(result.status_code)
self.app.logger.info("SEND CONFIRMED")
time.sleep(1)
@@ -254,9 +286,6 @@ async def query_database():
app.logger.info(record["query"])
app.logger.info(record["UUID"])
uuid = record["UUID"]
#TODO: TESTING ONLY======================
uuid = UUID('{12345678-1234-5678-1234-567812345678}')
cl = Librarian(app, record["query"], uuid)
librarian_queue.put(cl)
librarian_list.append(cl)
@@ -284,14 +313,15 @@ async def get_partial():
#=======================================MAIN===================================================
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger = app.logger
logger.setLevel(logging.DEBUG)
threads = []
#threads.append(threading.Thread(target=waitress_run))
threads.append(threading.Thread(target=flask_debug))
threads.append(threading.Thread(target=waitress_run))
#threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch()
bgtask.app = app
threads.append(bgtask)
threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,)))
i = 0
for worker in threads:
try:
conjurer_librarian/run_server.bat
+1
View File
@@ -0,0 +1 @@
c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py
conjurer_librarian/scrape_bot.py
+93
View File
@@ -0,0 +1,93 @@
import json
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from requests import Timeout,ConnectTimeout, ConnectionError
from queue import Queue
import re
import time
import random
import logging
from threading import Thread
SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
SCR_FILENAME = "40_chunk.txt"
SCR_ENCODING = "utf-8"
WORK_Q = Queue()
random.seed()
def load_ndb_to_q(logger):
logger.info("Loader started")
while True:
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
for item in ndb_database.keys():
logger.info(item)
url = f"https://sci-hub.se/{item}"
WORK_Q.put([item, url, False])
ndb_file.truncate(0)
except JSONDecodeError:
time.sleep(60*60*2)
pass
time.sleep(180)
def check_if_exists_brute_force(logger):
while True:
logger.info("Scraper tick")
item = WORK_Q.get()
page_url =item[1]
logger.error("REFINE: Brute force search!")
if not page_url.startswith(("http:", "https:")):
raise ValueError("URL must start with 'http:' or 'https:'")
blocked = True
try:
# trunk-ignore(bandit/B310)
with urlopen(page_url) as response:
data = response.read()
text = data.decode("utf-8")
if item[2]:
logger.info("Already found")
time.sleep(180)
blocked = False
else:
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
blocked = False
logger.info("Not found")
item[2] = False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
blocked = False
direct_download_link = "https:" + str(m.group(1))
logger.info (direct_download_link)
with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
operated_file.write("\n")
operated_file.write(item[0])
item[2] = True
except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
pass
if blocked:
logger.info(item)
logger.info(text)
logger.error("Got blocked. Fuck.")
time.sleep(60*60*72)
# trunk-ignore(bandit/B311)
rand = random.randint(1,100)
logger.info(f"Sleeping for {3*rand} minutes")
time.sleep(180*rand)
def scraper(logger = None):
if not logger:
logger = logging.getLogger()
logger.setLevel("DEBUG")
h1 = logging.StreamHandler()
logger.addHandler(h1)
logger.info("No logger set. Test run")
loader = Thread(target = load_ndb_to_q, args = (logger,))
scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
loader.start()
scraper.start()
loader.join()
scraper.join()
if __name__ == "__main__":
scraper()
conjurer_librarian/search_bot.py
+23 -23
View File
@@ -1,25 +1,30 @@
#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
from queue import Queue, Empty
from threading import Thread
q = Queue()
#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy
#MAXTHREADS = 3
MAXTHREADS = 1
#TODO: DATA FOR TEST ONLY
#MAXTHREADS = 5
#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
#TODO: DEPLOYMENT DATA
MAXTHREADS = 41
DATABASE_PATH = r'C:\\Database\\chunks\\'
ENCODING = "utf-8"
CHUNk = "_chunk.txt"
_sentinel = object()
result_list = []
WORK_Q_SIZE = 15500000
#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
def producer(out_q, control_q, filename):
def producer(out_q, control_q, filename, logger):
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
lines = operated_file.readlines()
for line in lines:
logger.info(f"Worker {filename} ")
while True:
line = operated_file.readline()
if not line:
break
out_q.put(line)
try:
check = control_q.get(block=False)
@@ -27,11 +32,11 @@ def producer(out_q, control_q, filename):
check = False
pass
if check is _sentinel:
print("TERM signal received")
logger.info("TERM signal received")
control_q.put(check)
break
print(f"Worker finished: {filename}")
out_q.put(_sentinel)
logger.info(f"Worker finished: {filename}")
out_q.put(_sentinel)
#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
@@ -40,10 +45,6 @@ def consumer(in_q, control_q, doi, live_results, logger):
for item in doi:
result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
while True:
#if in_q.empty():
# print("Queue empty!")
#if in_q.full():
# print("Queue full!")
done_check = True
data = in_q.get()
if data is _sentinel:
@@ -51,7 +52,7 @@ def consumer(in_q, control_q, doi, live_results, logger):
sentinels += 1
else:
for item in result_list:
if item["DOI"] in data:
if item["DOI"] in data and not item["exists"]:
logger.info(data)
logger.info("HIT")
item["exists"] = True
@@ -66,25 +67,24 @@ def consumer(in_q, control_q, doi, live_results, logger):
def search_for_doi(doi, live_results, logger):
threads = []
work_q = Queue(maxsize=200000)
work_q = Queue(maxsize=WORK_Q_SIZE)
control_q = Queue()
t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
logger.info("Consumer thread created")
threads.append(t_cons)
for i in range(0,MAXTHREADS):
#TODO: TEST DATA
#filename = "test" + str(i) + CHUNk
filename = "40" + CHUNk
#filename = str(i) + CHUNk
#TODO: DEPLOYMENT DATA
filename = str(i) + CHUNk
logger.info(f"Creating worker thread no: {i}")
threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
for worker in threads:
worker.start()
for worker in threads:
worker.join()
return result_list
if __name__ == "__main__":
import logging
logger = logging.getLogger()
cr_results.json
-1
View File
File diff suppressed because one or more lines are too long
file_webservice/conjurer_musician.py
+31 -11
View File
@@ -14,6 +14,7 @@ from sys import platform
import logging
from flask import Flask, jsonify, request
from waitress import serve
from logging import handlers
HOST_ADDRESS = "192.168.1.15"
@@ -26,21 +27,15 @@ if platform in ("linux", "linux2"):
NETRC_FILE = "/home/mtuszowski/.netrc"
LOGSTORE = "/home/mtuszowski/conjurer/logs/"
ENCODING = "utf-8"
GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
else:
LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
NETRC_FILE = "/home/pi/.netrc"
LOGSTORE = "/home/pi/RetroPie/logs/"
ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
ENCODING = "utf-8"
GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
music_file_list = []
@@ -188,6 +183,22 @@ def remove_characters(string, character):
return string.replace(character, "")
@app.route("/stream", methods=["GET"])
async def stream_music():
"""
The function `get_music_list` returns a JSON object containing a list of music files.
:return: A JSON response containing a key "music_file_list" with the value of the variable
`music_file_list`.
"""
try:
m3u8 = Path("/tmp/hls/stream.m3u8")
r = Response(m3u8)
r.headers["Content-Type"] = "application/mpegURL"
return r, 200
except FileNotFoundError:
return jsonify({"error": "No stream available"}), 404
@app.route("/mp3", methods=["GET"])
def get_music_list():
"""
@@ -261,6 +272,17 @@ def waitress_run():
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler = handlers.RotatingFileHandler(
filename=LOGFILE,
encoding=ENCODING,
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
)
handler.setFormatter(formatter)
logger.addHandler(handler)
rescan()
logger.info("Started")
threads = []
@@ -272,5 +294,3 @@ if __name__ == "__main__":
for worker in threads:
worker.join()
#TODO: Inne pliki - sadox, dokumenty naukowe itp.
file_webservice/conjurer_musician.service
+1 -1
View File
@@ -4,7 +4,7 @@ After=multi-user.target
[Service]
Type=simple
ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/conjurer_musician.py
Restart=on-abort
[Install]
not_in_db.json
View File
rr_results.json
-161654
View File
File diff suppressed because one or more lines are too long
s_results.json
-1
View File
@@ -1 +0,0 @@
{"12345678-1234-5678-1234-567812345678": [{"DOI": "10.1057/9781137435026", "title": ["Queer BDSM Intimacies"], "type": "book", "container": "NO"}]}