mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-15 05:48:35 +00:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a68745713b | |||
| bbe1b89007 | |||
| 27fefdccb5 | |||
| 64849f728f | |||
| 9b09c2c13f | |||
| f2fb2232ee | |||
| 7e4dfbf8cd | |||
| 81a387eb4f | |||
| 408f6e5c4e | |||
| a266e5d240 | |||
| 917b57c586 | |||
| 0e15218abd | |||
| f56773b2e1 | |||
| 944dfe2c18 | |||
| 2fd362a1d5 | |||
| 47945ab3dd | |||
| c0b362df6d | |||
| 575237812f | |||
| 60787d8162 | |||
| 8dc4eef149 | |||
| 0f49cb89a3 | |||
| 99f24e6b77 | |||
| 9b14610cc9 | |||
| 14df489b38 | |||
| 8a99ee7c99 |
@@ -0,0 +1,7 @@
|
||||
enable=all
|
||||
source-path=SCRIPTDIR
|
||||
disable=SC2154
|
||||
|
||||
# If you're having issues with shellcheck following source, disable the errors via:
|
||||
# disable=SC1090
|
||||
# disable=SC1091
|
||||
+15
-11
@@ -2,34 +2,38 @@
|
||||
# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
|
||||
version: 0.1
|
||||
cli:
|
||||
version: 1.20.1
|
||||
version: 1.22.2
|
||||
# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
|
||||
plugins:
|
||||
sources:
|
||||
- id: trunk
|
||||
ref: v1.4.5
|
||||
ref: v1.6.0
|
||||
uri: https://github.com/trunk-io/plugins
|
||||
# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
|
||||
runtimes:
|
||||
enabled:
|
||||
- go@1.21.0
|
||||
- node@18.12.1
|
||||
- python@3.10.8
|
||||
# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
|
||||
lint:
|
||||
enabled:
|
||||
- actionlint@1.6.27
|
||||
- bandit@1.7.8
|
||||
- black@24.3.0
|
||||
- shellcheck@0.10.0
|
||||
- shfmt@3.6.0
|
||||
- taplo@0.8.1
|
||||
- actionlint@1.7.1
|
||||
- bandit@1.7.9
|
||||
- black@24.4.2
|
||||
- codespell
|
||||
- checkov@3.2.48
|
||||
- checkov@3.2.139
|
||||
- git-diff-check
|
||||
- isort@5.13.2
|
||||
- pylint
|
||||
- markdownlint@0.39.0
|
||||
- prettier@3.2.5
|
||||
- ruff@0.3.4
|
||||
- trivy@0.50.1
|
||||
- trufflehog@3.71.0
|
||||
- markdownlint@0.41.0
|
||||
- prettier@3.3.2
|
||||
- ruff@0.4.9
|
||||
- trivy@0.52.2
|
||||
- trufflehog@3.78.1
|
||||
- yamllint@1.35.1
|
||||
actions:
|
||||
disabled:
|
||||
|
||||
@@ -19,6 +19,7 @@ import shutil
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from logging import handlers
|
||||
from pathlib import Path, PurePath
|
||||
@@ -135,10 +136,15 @@ MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15
|
||||
MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
|
||||
|
||||
FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
|
||||
RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321"
|
||||
SKIP_TRACK = "/skip"
|
||||
|
||||
GET_MP3 = "/mp3"
|
||||
SEND_MP3 = "/update_mp3"
|
||||
GET_PLAYLIST = "/get_music"
|
||||
ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
|
||||
CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
|
||||
|
||||
REQUEST_MUSIC = "/request_radio_file"
|
||||
CLEAR_PRIO = "/clear_pr_pls"
|
||||
LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
|
||||
@@ -929,17 +935,17 @@ async def handle_response(
|
||||
"Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size
|
||||
)
|
||||
if request_type == "MUSIC":
|
||||
algorithm = "gpt-3.5-turbo-0125"
|
||||
algorithm = "gpt-4o"
|
||||
table = message_table_muzyka
|
||||
token_amount = 10700
|
||||
elif request_type == "RANDOM":
|
||||
algorithm = "gpt-3.5-turbo-0125"
|
||||
algorithm = "gpt-4o"
|
||||
table = MESSAGE_TABLE
|
||||
token_amount = 10700
|
||||
else:
|
||||
table = MESSAGE_TABLE
|
||||
algorithm = "gpt-4"
|
||||
token_amount = 7000
|
||||
algorithm = "gpt-4o"
|
||||
token_amount = 10700
|
||||
|
||||
prompt_gpt_request_size = num_tokens_from_string(
|
||||
{"role": "user", "content": final_prompt}, "gpt-4"
|
||||
@@ -1345,6 +1351,46 @@ async def max_weight(lista):
|
||||
|
||||
# *=========================================== Define Commands
|
||||
|
||||
@client.hybrid_command(
|
||||
name="skip_track",
|
||||
description="Przeskocz kawałek w radiu",
|
||||
guild=discord.Object(id=664789470779932693),
|
||||
)
|
||||
async def skip_track(ctx):
|
||||
async with ctx.typing():
|
||||
allowed = False
|
||||
for role in ctx.author.roles:
|
||||
if role.name in ("Thane","Jarl"):
|
||||
allowed = True
|
||||
if not allowed:
|
||||
await ctx.send("Łapy precz od radia")
|
||||
else:
|
||||
coroutine = asyncio.to_thread(
|
||||
requests.get,
|
||||
f"{RADIO_HARBOR_ADDRESS}{SKIP_TRACK}",
|
||||
timeout=360,
|
||||
)
|
||||
result = await coroutine
|
||||
logger.info("Done %s", result)
|
||||
await ctx.send("Zrobione szefie!")
|
||||
|
||||
|
||||
@client.hybrid_command(
|
||||
name="zrestartuj_radio",
|
||||
description="Komenda ktora uruchamia radio ponownie jakby się zawiesiło",
|
||||
guild=discord.Object(id=664789470779932693),
|
||||
)
|
||||
async def zrestartuj_radio(ctx):
|
||||
async with ctx.typing():
|
||||
allowed = False
|
||||
for role in ctx.author.roles:
|
||||
if role.name in ("Thane","Jarl"):
|
||||
allowed = True
|
||||
if not allowed:
|
||||
await ctx.send("Łapy precz od radia")
|
||||
else:
|
||||
retcode = subprocess.run("/home/pi/Conjurer/restart_radio.sh", shell=False, check = False, capture_output = True)
|
||||
logger.info("Wynik: %s",retcode)
|
||||
|
||||
@client.hybrid_command(
|
||||
name="dej_co_ze_spotifaja",
|
||||
@@ -1795,7 +1841,9 @@ async def dodaj_do_ulubionych(ctx):
|
||||
json=jrequest,
|
||||
timeout=360,
|
||||
)
|
||||
_ = await coroutine
|
||||
result = await coroutine
|
||||
logger.info("Done %s", result)
|
||||
await ctx.send("Zrobione szefie!")
|
||||
@client.hybrid_command(
|
||||
name="ja_chciol",
|
||||
description="Dodaje do listy ulubionych w radiu",
|
||||
@@ -1825,7 +1873,46 @@ async def request_radio(ctx):
|
||||
json=jrequest,
|
||||
timeout=360,
|
||||
)
|
||||
_ = await coroutine
|
||||
result = await coroutine
|
||||
logger.info("Done %s", result)
|
||||
await ctx.send("Zrobione szefie!")
|
||||
|
||||
@client.hybrid_command(
|
||||
name="stworz_audycje",
|
||||
description="Dodaje do listy ulubionych w radiu",
|
||||
guild=discord.Object(id=664789470779932693),
|
||||
)
|
||||
async def stworz_audycje(ctx):
|
||||
"""
|
||||
Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
|
||||
Rest of the line are search terms.
|
||||
|
||||
:param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
|
||||
represents the context in which the command was invoked, including information such as the message,
|
||||
the channel, the server, and the user who invoked the command. This parameter is required in all
|
||||
Discord.py commands
|
||||
"""
|
||||
async with ctx.typing():
|
||||
logger.info("Zaczynam szukać timestamp %s", datetime.now())
|
||||
|
||||
dlugosc_playlisty = ctx.message.content.split()[1]
|
||||
word_list = ctx.message.content.split()
|
||||
jrequest = {
|
||||
"lista_slow": word_list,
|
||||
"dlugosc_plejlisty": dlugosc_playlisty,
|
||||
"UUID": str(uuid.uuid4()),
|
||||
}
|
||||
coroutine = asyncio.to_thread(
|
||||
requests.post,
|
||||
f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}",
|
||||
json=jrequest,
|
||||
timeout=360,
|
||||
)
|
||||
result = await coroutine
|
||||
logger.info("Done %s", result)
|
||||
await ctx.send("Zrobione szefie!")
|
||||
|
||||
|
||||
@client.hybrid_command(
|
||||
name="wyczysc_ulubione",
|
||||
description="Czysci liste ulubionych w radiu",
|
||||
@@ -1847,7 +1934,9 @@ async def wyczysc_ulubione(ctx):
|
||||
f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
|
||||
timeout=360,
|
||||
)
|
||||
_ = await coroutine
|
||||
result = await coroutine
|
||||
logger.info("Done %s", result)
|
||||
await ctx.send("Zrobione szefie!")
|
||||
|
||||
|
||||
@client.hybrid_command(
|
||||
@@ -2374,7 +2463,7 @@ async def krecimy_pornola(ctx):
|
||||
sciezka, files = await get_file(ctx, "Pornol", item_yt)
|
||||
if files:
|
||||
logger.info("Pornol udany")
|
||||
ctx.reply(f"Jest w tajnym archiwum pod adresem{sciezka})")
|
||||
ctx.send(f"Jest w tajnym archiwum pod adresem{sciezka})")
|
||||
else:
|
||||
ctx.reply("Jak ładnie szefa poprosisz.")
|
||||
|
||||
@@ -2399,10 +2488,3 @@ if __name__ == "__main__":
|
||||
WRK_CNT+=1
|
||||
logger.info("Starting discord bot: Joining thread %s", WRK_CNT)
|
||||
worker.join()
|
||||
|
||||
# widok - generuje się 5 stron po 4 linki
|
||||
# odśwież - zamienić na "dalej" - zaciąga kolejne 20 rezultatów
|
||||
# po wyslaniu czekamy na odbior i dodajemy UUID oraz nick poszukiwacza do queue ktora jest sprawdzana
|
||||
# flask sobie chodzi i jak dostaje posta z wynikami to wyciaga z queue UUID i odpowiada że znalazła
|
||||
# po samym wyszukaniu dajemy info że "Zaczynam szukać ale pare godzin to potrwa."
|
||||
# OPCJONALNIE: Dosyłamy znalezione wyniki na bieżąco - wtedy wkładamy do queue dla UUID wyniki
|
||||
|
||||
@@ -26,6 +26,7 @@ from queue import Queue
|
||||
import requests
|
||||
import scrape_bot
|
||||
import search_bot
|
||||
#import search_bot2 as search_bot
|
||||
from flask import Flask, jsonify, request
|
||||
from habanero import Crossref
|
||||
from waitress import serve
|
||||
@@ -37,7 +38,10 @@ PORT_ADDRESS = 5001
|
||||
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
|
||||
SEND_RESULTS = "/conjurer"
|
||||
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
|
||||
MAX_CR_RESULTS = 7000
|
||||
|
||||
MAX_CR_RESULTS = 1000
|
||||
#TEST PURPOSES ONLY!
|
||||
#MAX_CR_RESULTS = 5
|
||||
|
||||
ENCODING = "utf-8"
|
||||
|
||||
@@ -114,14 +118,15 @@ class Librarian(object):
|
||||
self.app.logger.info("STARTED SEARCH")
|
||||
|
||||
if not deep_search:
|
||||
cr_result = self.cr.works(query=query, limit=1000)
|
||||
query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000
|
||||
cr_result = self.cr.works(query=query, limit=query_limit)
|
||||
self.search_result_from_cr.update(cr_result)
|
||||
self.total = cr_result["message"]["total-results"]
|
||||
self.fetched += len(cr_result["message"]["items"])
|
||||
self.app.logger.info(self.total)
|
||||
self.app.logger.info(self.fetched)
|
||||
while self.total > self.fetched and self.limit > self.fetched:
|
||||
tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched)
|
||||
tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched)
|
||||
cr_result["message"]["items"].extend(tmp_result["message"]["items"])
|
||||
self.total = tmp_result["message"]["total-results"]
|
||||
self.fetched = len(cr_result["message"]["items"])
|
||||
@@ -130,7 +135,7 @@ class Librarian(object):
|
||||
time.sleep(0.1)
|
||||
|
||||
else:
|
||||
cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True)
|
||||
cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
|
||||
result = cr_result[0]
|
||||
for item in cr_result[1:]:
|
||||
result["message"]["items"].extend(item["message"]["items"])
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
|
||||
FILEPTH = r"C:\Database\chunks\4_chunk.txt"
|
||||
|
||||
with open(FILEPTH, "r", encoding="utf-8") as file:
|
||||
tab = file.readlines()
|
||||
print(len(tab))
|
||||
|
||||
#1842295
|
||||
@@ -33,12 +33,14 @@ def load_ndb_to_q(logger):
|
||||
try:
|
||||
ndb_database = json.load(ndb_file)
|
||||
for _ in range (1,10):
|
||||
key = next(iter(ndb_database))
|
||||
item = ndb_database.pop(key)
|
||||
logger.info(key)
|
||||
url = f"https://sci-hub.se/{key}"
|
||||
WORK_Q.put([item, url, False])
|
||||
|
||||
try:
|
||||
key = next(iter(ndb_database))
|
||||
_ = ndb_database.pop(key)
|
||||
logger.info(key)
|
||||
url = f"https://sci-hub.se/{key}"
|
||||
WORK_Q.put([key, url, False])
|
||||
except StopIteration:
|
||||
break
|
||||
ndb_file.truncate(0)
|
||||
ndb_file.seek(0)
|
||||
json.dump(ndb_database, ndb_file, indent=4)
|
||||
|
||||
@@ -21,8 +21,9 @@ Global Variables:
|
||||
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
|
||||
from queue import Empty, Queue
|
||||
from threading import Thread
|
||||
|
||||
import time
|
||||
q = Queue()
|
||||
#TODO: Count number of lines in files and print to approximate on which part of the file search is
|
||||
|
||||
# DATA FOR TEST ONLY
|
||||
# MAXTHREADS = 5
|
||||
@@ -49,9 +50,17 @@ def producer(out_q, control_q, filename, _logger):
|
||||
_logger: Logger object for logging.
|
||||
"""
|
||||
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
|
||||
_logger.info(f"Worker {filename} ")
|
||||
print(f"Worker {filename} ")
|
||||
line_no = 0
|
||||
while True:
|
||||
line = operated_file.readline()
|
||||
line_no += 1
|
||||
print(f"\t \t \t \t \t \t W{filename}{line_no}\r", end="")
|
||||
|
||||
if not line:
|
||||
print(f"EOF {filename}")
|
||||
break
|
||||
|
||||
if not line:
|
||||
break
|
||||
out_q.put(line)
|
||||
@@ -59,11 +68,12 @@ def producer(out_q, control_q, filename, _logger):
|
||||
check = control_q.get(block=False)
|
||||
except Empty:
|
||||
check = False
|
||||
|
||||
if check is _sentinel:
|
||||
_logger.info("TERM signal received")
|
||||
print("TERM signal received")
|
||||
control_q.put(check)
|
||||
break
|
||||
_logger.info(f"Worker finished: {filename}")
|
||||
print(f"Worker finished: {filename}")
|
||||
out_q.put(_sentinel)
|
||||
|
||||
|
||||
@@ -79,19 +89,25 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
|
||||
live_results (list): List to store the search results.
|
||||
_logger: Logger object for logging.
|
||||
"""
|
||||
_logger.info("Consumer thread started:%s", no)
|
||||
for item in doi:
|
||||
result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
|
||||
print(f"Consumer thread started: {no} no")
|
||||
empty_counter = 0
|
||||
alive_no = 0
|
||||
while True:
|
||||
done_check = True
|
||||
try:
|
||||
data = in_q.get(block=False)
|
||||
data = in_q.get(block=True, timeout = 1)
|
||||
if data is _sentinel:
|
||||
_logger.info("Worker finished")
|
||||
control_dict["sentinels"] += 1
|
||||
print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
|
||||
|
||||
else:
|
||||
empty_counter = 0
|
||||
alive_no += 1
|
||||
print(f"C{no}__{alive_no}\r", end="")
|
||||
|
||||
for item in result_list:
|
||||
if item["DOI"] in data and not item["exists"]:
|
||||
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
|
||||
_logger.info(data)
|
||||
_logger.info("HIT")
|
||||
item["exists"] = True
|
||||
@@ -100,9 +116,18 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
|
||||
if done_check:
|
||||
control_q.put(_sentinel)
|
||||
except Empty:
|
||||
pass
|
||||
empty_counter += 1
|
||||
time.sleep(1)
|
||||
print(f"Consumer {no} empty")
|
||||
if empty_counter > 5:
|
||||
print("Consumer %s empty lvl 2", no)
|
||||
time.sleep(2)
|
||||
elif empty_counter > 10:
|
||||
print(f"Consumer thread finished {no}")
|
||||
break
|
||||
|
||||
if control_dict["sentinels"] >= MAXTHREADS:
|
||||
_logger.info("All workers finished")
|
||||
_logger.info(f"All workers finished {no}")
|
||||
break
|
||||
|
||||
|
||||
@@ -121,6 +146,10 @@ def search_for_doi(doi, live_results, _logger):
|
||||
threads = []
|
||||
work_q = Queue(maxsize=WORK_Q_SIZE)
|
||||
control_q = Queue()
|
||||
|
||||
for item in doi:
|
||||
result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
|
||||
|
||||
for i in range (0, (len(doi)//1000)+2):
|
||||
t_cons = Thread(
|
||||
target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
"""
|
||||
This module contains functions for searching for DOI (Digital Object Identifier)
|
||||
in a list of live results.
|
||||
It includes a producer-consumer pattern implementation using threads and queues.
|
||||
|
||||
Functions:
|
||||
- producer: Reads lines from a file and puts them into an output queue.
|
||||
- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
|
||||
- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
|
||||
|
||||
Global Variables:
|
||||
- MAXTHREADS: Maximum number of worker threads.
|
||||
- DATABASE_PATH: Path to the database files.
|
||||
- ENCODING: Encoding of the database files.
|
||||
- CHUNK: File name pattern for the database files.
|
||||
- _sentinel: Sentinel object used to signal termination.
|
||||
- result_list: List to store the search results.
|
||||
- WORK_Q_SIZE: Maximum size of the work queue.
|
||||
"""
|
||||
|
||||
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
|
||||
from multiprocessing import Manager, Process, Queue
|
||||
from queue import Empty, Full
|
||||
import time
|
||||
|
||||
# DATA FOR TEST ONLY
|
||||
#MAXTHREADS = 6
|
||||
#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
|
||||
|
||||
# DEPLOYMENT DATA
|
||||
MAXTHREADS = 41
|
||||
DATABASE_PATH = r"C:\\Database\\chunks\\"
|
||||
|
||||
ENCODING = "utf-8"
|
||||
CHUNK = "_chunk.txt"
|
||||
_sentinel = None
|
||||
WORK_Q_SIZE = 355000
|
||||
|
||||
|
||||
def producer(out_q, control_q, filename, _logger):
|
||||
"""
|
||||
Produces items from the output queue and puts them into the control queue.
|
||||
|
||||
Args:
|
||||
out_q (Queue): Output queue.
|
||||
control_q (Queue): Control queue.
|
||||
filename (str): Name of the file.
|
||||
_logger: Logger object for logging.
|
||||
"""
|
||||
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
|
||||
print(f"Worker {filename} ")
|
||||
line_no = 0
|
||||
while True:
|
||||
line = operated_file.readline()
|
||||
line_no += 1
|
||||
print(f"\t \t \t W{filename}{line_no}\r", end="")
|
||||
|
||||
if not line:
|
||||
print(f"EOF {filename}")
|
||||
break
|
||||
try:
|
||||
out_q.put([line, line_no, filename], block = True, timeout = 60)
|
||||
except Full:
|
||||
print("queue fulll")
|
||||
try:
|
||||
check = control_q.get(block=False)
|
||||
except Empty:
|
||||
check = False
|
||||
if check is None:
|
||||
print(f"TERM signal received {filename}")
|
||||
control_q.put(check)
|
||||
control_q.close()
|
||||
break
|
||||
print(f"Worker finished: {filename}")
|
||||
out_q.put(_sentinel)
|
||||
out_q.close()
|
||||
|
||||
|
||||
|
||||
# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
|
||||
def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
|
||||
"""
|
||||
Consumes items from an input queue and checks if DOI exists in the live results list.
|
||||
|
||||
Args:
|
||||
in_q (Queue): Input queue.
|
||||
control_q (Queue): Control queue.
|
||||
doi (list): List of DOI to search for.
|
||||
live_results (list): List to store the search results.
|
||||
_logger: Logger object for logging.
|
||||
"""
|
||||
print(f"Consumer thread started: {no} no")
|
||||
empty_counter = 0
|
||||
alive_no = 0
|
||||
while True:
|
||||
done_check = True
|
||||
try:
|
||||
data = in_q.get(block=True, timeout = 1)
|
||||
empty_counter = 0
|
||||
if data is None:
|
||||
control_dict["sentinels"] += 1
|
||||
print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
|
||||
else:
|
||||
empty_counter = 0
|
||||
if alive_no > 9:
|
||||
alive_no = 0
|
||||
alive_no += 1
|
||||
print(f"C{no}{alive_no}\r", end="")
|
||||
for item in result_list:
|
||||
if item["DOI"] in data[0] and not item["exists"]:
|
||||
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
|
||||
item["exists"] = True
|
||||
live_results.append(item)
|
||||
done_check = done_check and item["exists"]
|
||||
if done_check:
|
||||
control_q.put(_sentinel)
|
||||
except Empty:
|
||||
empty_counter += 1
|
||||
time.sleep(1)
|
||||
print(f"Consumer {no} empty")
|
||||
if empty_counter > 5:
|
||||
print("Consumer %s empty lvl 2", no)
|
||||
time.sleep(2)
|
||||
elif empty_counter > 10:
|
||||
print(f"Consumer thread finished {no}")
|
||||
break
|
||||
if control_dict["sentinels"] >= MAXTHREADS:
|
||||
print(f"All workers finished killing {no}")
|
||||
break
|
||||
print("Consumer thread finished %s",no )
|
||||
control_q.close()
|
||||
|
||||
|
||||
def search_for_doi(doi, live_results, _logger):
|
||||
"""
|
||||
Search for DOI in live_results using _logger for logging.
|
||||
|
||||
Args:
|
||||
doi (list): List of DOI to search for.
|
||||
live_results (list): List to store the search results.
|
||||
_logger: Logger object for logging.
|
||||
"""
|
||||
with Manager() as manager:
|
||||
control_dict = manager.dict()
|
||||
control_dict["sentinels"] = 0
|
||||
result_list = []
|
||||
for item in doi:
|
||||
result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
|
||||
result_list_proxy = manager.list(result_list)
|
||||
|
||||
threads = []
|
||||
work_q = Queue(maxsize=WORK_Q_SIZE)
|
||||
control_q = Queue()
|
||||
for i in range(0, MAXTHREADS):
|
||||
# TEST DATA
|
||||
#filename = "test" + str(i) + CHUNK
|
||||
# DEPLOYMENT DATA
|
||||
filename = str(i) + CHUNK
|
||||
print(f"Creating worker thread no: {i}")
|
||||
threads.append(
|
||||
Process(target=producer, args=(work_q, control_q, filename, _logger))
|
||||
)
|
||||
for i in range (0, (len(doi)//1000)+5):
|
||||
t_cons = Process(
|
||||
target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger)
|
||||
)
|
||||
print("Consumer thread created")
|
||||
threads.append(t_cons)
|
||||
|
||||
for worker in threads:
|
||||
worker.start()
|
||||
for worker in threads:
|
||||
worker.join()
|
||||
result_list = list(result_list_proxy)
|
||||
for worker in threads:
|
||||
worker.close()
|
||||
return result_list
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel("DEBUG")
|
||||
h1 = logging.StreamHandler()
|
||||
logger.addHandler(h1)
|
||||
print("TEST RUN")
|
||||
live_result = []
|
||||
print(
|
||||
search_for_doi(
|
||||
[
|
||||
("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
|
||||
("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
|
||||
("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
|
||||
("10.2165/00128415-200309690-00017", "DATA"),
|
||||
("10.5772/48313", "DATA"),
|
||||
("10.15803/ijnc.7.2_419", "DATA"),
|
||||
("10.1051/0004-6361/201321596e", "DATA"),
|
||||
("10.2307/40835941", "DATA"),
|
||||
("NOT_ON_LIST", "DATA"),
|
||||
],
|
||||
live_result,
|
||||
logger,
|
||||
)
|
||||
)
|
||||
print(live_result)
|
||||
@@ -206,6 +206,7 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
|
||||
fun_logger.info("Wiele plików do zagrania")
|
||||
search_weight.sort(key=lambda x: x[0], reverse=True)
|
||||
return_list.extend(search_weight[: int(how_many)])
|
||||
fun_logger.info("Done: %s", return_list)
|
||||
return return_list
|
||||
|
||||
|
||||
@@ -369,6 +370,37 @@ def add_request():
|
||||
)
|
||||
return return_data
|
||||
|
||||
@app.route("/create_priority_playlist", methods=["POST"])
|
||||
def create_priority_playlist():
|
||||
"""
|
||||
The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received
|
||||
item, adds it to a music file list, and returns a success message along with the updated record.
|
||||
|
||||
:return: A tuple containing a JSON response and a status code.
|
||||
The JSON response includes keys `isError`,
|
||||
`message`, `statusCode`, and `data`, with values
|
||||
indicating the success of the operation and the
|
||||
data that was received and added to the `music_file_list`.
|
||||
The status code returned is 200,indicating a successful response.
|
||||
"""
|
||||
record = json.loads(request.data)
|
||||
app.logger.info(record)
|
||||
app.logger.info(record["lista_slow"])
|
||||
app.logger.info(record["UUID"])
|
||||
app.logger.info(record["dlugosc_plejlisty"])
|
||||
return_data = wyszukaj(
|
||||
record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
|
||||
)
|
||||
with open(
|
||||
"/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
|
||||
) as s_file:
|
||||
for item in return_data:
|
||||
s_file.write(item[1] + "\n")
|
||||
return_data = (
|
||||
jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
|
||||
200,
|
||||
)
|
||||
return return_data
|
||||
|
||||
|
||||
@app.route("/add_to_priority", methods=["POST"])
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
$dodaj_do_ulubionych 100 nightwish
|
||||
$dodaj_do_ulubionych 100 amon amarth
|
||||
$dodaj_do_ulubionych 100 tactical sekt
|
||||
$dodaj_do_ulubionych 100 lacuna coil
|
||||
$dodaj_do_ulubionych 100 sefa
|
||||
$dodaj_do_ulubionych 100 frenchcore
|
||||
$dodaj_do_ulubionych 100 peacock
|
||||
$dodaj_do_ulubionych 100 vavamuffin
|
||||
$dodaj_do_ulubionych 100 abradab
|
||||
$dodaj_do_ulubionych 100 eluveitie
|
||||
$dodaj_do_ulubionych 100 youtube
|
||||
|
||||
$dodaj_do_ulubionych 100 elyose
|
||||
$dodaj_do_ulubionych 1 artisans du chaos
|
||||
@@ -34,10 +34,9 @@ requests_queue = request.queue()
|
||||
|
||||
# Function to process the request queue and add new requests
|
||||
def queue_processing()
|
||||
f1=file.read("/home/pi/Conjurer/request.playlist")
|
||||
text = f1()
|
||||
if text != "" then
|
||||
requests_queue.push.uri(text)
|
||||
text=file.lines("/home/pi/Conjurer/request.playlist")
|
||||
if text != [] then
|
||||
list.iter(fun(item) -> requests_queue.push.uri(item), text)
|
||||
file.remove("/home/pi/Conjurer/request.playlist")
|
||||
f = file.open("/home/pi/Conjurer/request.playlist", create=true)
|
||||
f.close()
|
||||
@@ -45,7 +44,7 @@ def queue_processing()
|
||||
end
|
||||
|
||||
# Randomly select a playlist with weights
|
||||
s4 = random(id="randomizer", weights=[1, 3, 5], [s1, s2, s3])
|
||||
s4 = random(id="randomizer", weights=[2, 3, 5], [s1, s2, s3])
|
||||
|
||||
# Load jingles playlist
|
||||
jingles = (playlist(reload_mode="watch", "/home/pi/Conjurer/jingles.playlist"))
|
||||
@@ -74,7 +73,7 @@ s = amplify(a, s)
|
||||
|
||||
mic = buffer(input.pulseaudio()) # Microphone
|
||||
mic = gate(threshold=-30., range=-80., mic)
|
||||
mic = blank.strip(max_blank=2., min_noise=.1, threshold=-20., mic)
|
||||
mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic)
|
||||
|
||||
s = add([s, mic])
|
||||
#w = interactive.float("wetness", min=0., max=1., 1.)
|
||||
@@ -82,7 +81,7 @@ s = add([s, mic])
|
||||
|
||||
|
||||
# Skip blank sections in the stream
|
||||
s = blank.skip(max_blank=2., s)
|
||||
s = blank.skip(max_blank=10., s)
|
||||
|
||||
# Configure logging settings
|
||||
log_to_stdout = true
|
||||
@@ -102,6 +101,13 @@ radio = fallback(id="switcher2", track_sensitive=false, [s, emergency])
|
||||
# Set up an interactive control for skipping tracks
|
||||
p = interactive.bool("Skip track", false)
|
||||
|
||||
def http_skip(~protocol, ~data, ~headers, uri)=
|
||||
radio.skip()
|
||||
http.response(code=200,data="Skipped")
|
||||
end
|
||||
harbor.http.register(port=54321,method="GET","/skip", http_skip)
|
||||
|
||||
|
||||
# Function to process the request queue and skip the current track if requested
|
||||
def check_skip()
|
||||
if p() then
|
||||
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
sshpass -p raspberry ssh pi@192.168.1.15 << 'ENDSSH'
|
||||
sudo reboot
|
||||
ENDSSH
|
||||
Reference in New Issue
Block a user