diff --git a/bot.py b/bot.py index 3c6167c..2548624 100644 --- a/bot.py +++ b/bot.py @@ -131,43 +131,6 @@ class MusicFileList(object): post_data = {"item": str(item)} requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360) -class DoSearchView(discord.ui.View): - def __init__(self, _uuid, result_list, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.uuid = _uuid - self.result_list = result_list - - @discord.ui.button( - label="Nastepna strona", style=discord.ButtonStyle.primary, row=0 - ) - async def next_callback(self, interaction, button): - # emoji = await ctx.guild.fetch_emoji(id) - # button.emoji=1102174413887111189 - logger.info("Pressed") - await interaction.response.edit_message( - content="You clicked the button!" - ) # Send a message when the button is clicked - - @discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0) - async def reset_callback(self, interaction, button): - # emoji = await ctx.guild.fetch_emoji(id) - # button.emoji=1102174413887111189 - logger.info("Pressed") - await interaction.response.edit_message( - content="You clicked the button!" - ) # Send a message when the button is clicked - - @discord.ui.button( - label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0 - ) - async def perv_callback(self, interaction, button): - # emoji = await ctx.guild.fetch_emoji(id) - # button.emoji=1102174413887111189 - logger.info("Pressed") - await interaction.response.edit_message( - content="You clicked the button!" - ) # Send a message when the button is clicked - # *=========================================== Predefines formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -351,9 +314,6 @@ async def on_ready(): @client.event -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0914) -# trunk-ignore(pylint/R0912) async def on_message(message): """ Handle incoming messages in a Discord server, perform various @@ -607,8 +567,6 @@ async def disconnect(ctx): await voice_client.disconnect() -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0912) async def play(ctx, zamawial=None, arg=None): """ Play a music file, retrieve metadata about the file, and send a @@ -728,24 +686,26 @@ async def check_music(): await asyncio.sleep(2) @tasks.loop(seconds=3) -# trunk-ignore(pylint/R0915) async def check_data_q(): + """ + This function checks the data queue for any new entries and processes them accordingly. + """ try: - data = IN_COMM_Q.get(block=False) + fresh_data = IN_COMM_Q.get(block=False) entries = [] - if data.stop: - searcher = data.author - ctx = data.ctx - query = data.content + if fresh_data.stop: + searcher = fresh_data.author + ctx = fresh_data.ctx + query = fresh_data.content l_p = 1 - for doi in data.entries: + for doi in fresh_data.entries: logger.info(doi) - desc = data.entries[doi] + desc = fresh_data.entries[doi] title = desc['Title'][0] entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n") l_p+=1 message = '*Z podłogi wysuwa się winda na książki*' - message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query}' + message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}' message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku ' if len(entries)<1: message += "niestety nie ma nic" @@ -768,13 +728,11 @@ async def check_data_q(): pass @tasks.loop(seconds=120) -# trunk-ignore(pylint/R0915) async def check_self(): """ The function `check_data` periodically checks for conditions to send messages and manage log files in a Discord channel. """ - # trunk-ignore(codespell/misspelled) # logger.info("Heartbeat of cleanup proc") channel = client.get_channel(1062047367337095268) messages = [message async for message in channel.history(limit=1)] @@ -837,9 +795,7 @@ async def check_self(): await channel.send( "*Przeciąga się za barem* No dobra - porobione to można dalej pi... *Zauważa spojrzenie Hammera* eprzyć o głupotach z klientami... Szefie... *Bierze 'ukradkowy' łyk z piersiówki*" ) - # trunk-ignore(pylint/W0603) global LAST_SPONTANEOUS_CALL - # trunk-ignore(pylint/W0603) global TIME_BETWEEN_CALLS tdelta = datetime.now() - LAST_SPONTANEOUS_CALL tdelta = tdelta.total_seconds() @@ -875,7 +831,6 @@ async def get_random_cyclic_message(): mess_key = list(cyclic_words.keys())[messnum] return cyclic_words[mess_key][0] ai_check2 = random.randint(0, 10) - # trunk-ignore(pylint/W0603) global MESSAGE_TABLE if ai_check2 < 6: logger.info("Dykteryjka") @@ -909,10 +864,6 @@ async def get_random_cyclic_message(): return result -# trunk-ignore(pylint/R0912) -# trunk-ignore(pylint/R0913) -# trunk-ignore(pylint/R0914) -# trunk-ignore(pylint/R0915) async def handle_response( prompt, vykidailo, bartender, history, username, request_type ): @@ -1101,8 +1052,6 @@ async def handle_response( return result, MESSAGE_TABLE -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0914) async def get_file(ctx, source, link): """ Take in three parameters: "ctx" (context), @@ -1323,9 +1272,6 @@ async def get_stats(ctx, history_limit): yield name, how_many -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0912) -# trunk-ignore(pylint/R0914) async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None): """ Take in a context object and an optional integer parameter "how_many" and perform search @@ -1485,8 +1431,6 @@ async def dej_co_z_jutuba(ctx): description="Jeśli nie wiesz jak użyć tej komendy to nawet nie próbuj", guild=discord.Object(id=664789470779932693), ) -# trunk-ignore(pylint/R0914) -# trunk-ignore(pylint/R0912) async def update_banlist(ctx): """ Update a banlist in a Discord guild from preprovided list in channel. @@ -2058,8 +2002,6 @@ async def get_image_stable_diffusion(ctx): description="Gra muzyke wyszukana na bazie tematow konwersacji na kanale NZ", guild=discord.Object(id=664789470779932693), ) -# trunk-ignore(pylint/R0912) -# trunk-ignore(pylint/R0915) async def zagraj_muzyke_mego_ludu(ctx): """ Play completeley random playlist based on messaging history. @@ -2209,11 +2151,20 @@ async def parametry_muzyki_mego_ludu( guild=discord.Object(id=664789470779932693), ) async def wyszukaj_linki_do_dokumentow(ctx): + """ + The function `wyszukaj_linki_do_dokumentow` searches for links to documents in a crossref database and provides links to scihub. + + :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots. + It represents the context of the command being executed, including information about the message, the server, + and the user who invoked the command. + """ query = ctx.message.content query_uuid = uuid.uuid4() #TODO: TESTING ONLY!! #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}') - json_query = {"UUID": str(query_uuid), "query": str(query)[30:], "page": 1} + ctx.message.content = ctx.message.content.replace("$wyszukaj_linki_do_dokumentow", "") + + json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": False} coroutine = asyncio.to_thread( requests.post, f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", @@ -2222,8 +2173,8 @@ async def wyszukaj_linki_do_dokumentow(ctx): ) await ctx.send( "*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" + - " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to te 3/4 stacji."+ - " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc te 12/8 stacji teraz będzie ciężko pracować" + " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."+ + " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować" ) query_response = await coroutine if not query_response.status_code == 200: @@ -2245,6 +2196,87 @@ async def wyszukaj_linki_do_dokumentow(ctx): await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+ " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.") +@client.hybrid_command( + name="glebokie_gardlo", + description="Przygotowuje drinka o nazwie głębokie gardło", + guild=discord.Object(id=664789470779932693), +) +async def wyszukaj_linki_do_dokumentow_deep(ctx): + """ + The function `wyszukaj_linki_do_dokumentow_deep` performs a deep search for links to documents in + a crossref database and provides links to scihub. + + :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots. + It represents the context of the command being executed, including information about the message, the server, + and the user who invoked the command. + """ + # TODO: Implement deep search logic here + query = ctx.message.content.replace("$glebokie_gardlo", "") + allowed = False + for role in ctx.message.author.roles: + if role.name == "Bartender": + allowed = True + if role.name == "Scribe": + allowed = True + if role.name == "Thane": + allowed = True + if not allowed: + if ctx.message.author.nick: + username = ctx.message.author.nick + else: + username = ctx.message.author.name + vykidailo = False + bartender = False + prompt = "Przygotuj mi drinka o nazwie Głębokie Gardło, inspirowanego tym sławnym filmem oraz skandalem Watergate" + for role in ctx.message.author.roles: + if role.name == "Vykidailo": + vykidailo = True + if role.name == "Bartender": + bartender = True + global MESSAGE_TABLE # pylint: disable=global-statement + + result, MESSAGE_TABLE = await handle_response( + prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION" + ) + await ctx.send(result) + return + + query_uuid = uuid.uuid4() + #TODO: TESTING ONLY!! + #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}') + json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": True} + coroutine = asyncio.to_thread( + requests.post, + f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", + json=json_query, + timeout=360, + ) + await ctx.send( + "*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" + + " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"+ + " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego" + ) + query_response = await coroutine + if not query_response.status_code == 200: + await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+ + " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało") + return + + query, query_uuid, queue_size = ( + query_response.json()["data"][0], + query_response.json()["data"][1], + query_response.json()["data"][2], + ) + if ctx.message.author.nick: + username = ctx.message.author.nick + else: + username = ctx.message.author.name + query_object = QueryControl(username, query_uuid, query, logger, ctx) + OUT_COMM_Q.put(query_object) + await ctx.send(f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."+ + " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*") + + @client.hybrid_command( name="krecimy_pornola", @@ -2289,16 +2321,16 @@ if __name__ == "__main__": threads.append(threading.Thread(target=client.run, args=(TOKEN,))) threads.append(threading.Thread(target=comm_subroutine, args=(logger,))) logger.info("Starting discord bot: Starting threads") - i = 0 + WRK_CNT = 0 for worker in threads: - i+=1 - logger.info("Starting discord bot: Starting thread {i}") + WRK_CNT+=1 + logger.info("Starting discord bot: Starting thread {WRK_CNT}") worker.start() logger.info("Starting discord bot: Joining threads") - i=0 + WRK_CNT=0 for worker in threads: - i+=1 - logger.info("Starting discord bot: Joining thread {i}") + WRK_CNT+=1 + logger.info("Starting discord bot: Joining thread {WRK_CNT}") worker.join() # widok - generuje się 5 stron po 4 linki diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index e3595e4..dc4c522 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -1,26 +1,27 @@ """ -This module contains the implementation of the Librarian class and related functions for searching and refining queries. +This module contains the implementation of the Librarian class and +related functions for searching and refining queries. Classes: -- Librarian: Represents a librarian object that performs search and refinement operations on queries. +- Librarian: Represents a librarian object that performs search and + refinement operations on queries Functions: -- flask_debug: Starts a Flask application in debug mode without using the reloader. -- waitress_run: Serves the Flask application using the Waitress WSGI server. -- BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously. +- flask_debug: Starts a Flask application in debug mode without using the reloader. +- waitress_run: Serves the Flask application using the Waitress WSGI server. +- BackgroundTaskSearch: Represents a background task for running the + Librarian object asynchronously """ import asyncio import json import logging import netrc -import re import threading import time from json.decoder import JSONDecodeError from logging import handlers from queue import Queue -from urllib.request import urlopen import requests import scrape_bot @@ -36,7 +37,8 @@ PORT_ADDRESS = 5001 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" SEND_RESULTS = "/conjurer" BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147" -MAX_CR_RESULTS = 150000 +MAX_CR_RESULTS = 7000 + ENCODING = "utf-8" app = Flask(__name__) @@ -45,8 +47,13 @@ librarian_queue = Queue() librarian_list = [] +# trunk-ignore(pylint/R0902) class Librarian(object): - def __init__(self, _app, query, uuid) -> None: + """ + Represents a librarian object that performs search and refinement operations on queries. + """ + + def __init__(self, _app, query, uuid, _deep_search) -> None: """ Initializes a Librarian object. @@ -71,8 +78,11 @@ class Librarian(object): - done: A flag indicating if the search is done. """ netrc_mod = netrc.netrc(NETRC_FILE) - authTokens = netrc_mod.authenticators("crossref") - self.cr = Crossref(mailto=authTokens[0]) + auth_tokens = netrc_mod.authenticators("crossref") + self.cr = Crossref( + mailto=auth_tokens[0], + ua_string=f"Conjurer project. mailto:{auth_tokens[0]}" + ) self.query = query self.uuid = str(uuid) self.limit = MAX_CR_RESULTS @@ -81,12 +91,13 @@ class Librarian(object): self.total = 0 self.app = _app self.live_results = [] - self.final_result = [] - self.not_in_db = [] + self.final_result = {} + self.not_in_db = {} self.search_result_from_cr = {} self.done = False + self.deep_search = _deep_search - async def search_crossref(self, query): + async def search_crossref(self, query, deep_search=False): """ Performs a search on Crossref for the given query. @@ -99,122 +110,124 @@ class Librarian(object): Raises: - None. """ + self.app.logger.info("STARTED SEARCH") - result = self.cr.works(query=query, limit=1000) - self.search_result_from_cr.update(result) - self.total = result["message"]["total-results"] - self.fetched += len(result["message"]["items"]) - self.app.logger.info(self.total) - self.app.logger.info(self.fetched) - while self.total > self.fetched and self.limit > self.fetched: - result = self.cr.works(query=query, limit=1000) - self.search_result_from_cr.update(result) - self.total = result["message"]["total-results"] - self.fetched += len(result["message"]["items"]) + + if not deep_search: + cr_result = self.cr.works(query=query, limit=1000) + self.search_result_from_cr.update(cr_result) + self.total = cr_result["message"]["total-results"] + self.fetched += len(cr_result["message"]["items"]) self.app.logger.info(self.total) self.app.logger.info(self.fetched) - time.sleep(0.1) - self.app.logger.info("CROSSREF DONE") - # TODO: add to sql with UUID - # sql row - title, doi, author, uuid, last retrieved, is_available + while self.total > self.fetched and self.limit > self.fetched: + tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched) + cr_result["message"]["items"].extend(tmp_result["message"]["items"]) + self.total = tmp_result["message"]["total-results"] + self.fetched = len(cr_result["message"]["items"]) + self.app.logger.info(self.total) + self.app.logger.info(self.fetched) + time.sleep(0.1) + else: + cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True) + result = cr_result[0] + for item in cr_result[1:]: + result["message"]["items"].extend(item["message"]["items"]) + self.total = item["message"]["total-results"] + self.fetched = len(result["message"]["items"]) + self.app.logger.info(self.total) + self.app.logger.info(self.fetched) + cr_result = result + self.app.logger.info("Total, fetched:") + self.app.logger.info(self.total) + self.app.logger.info(self.fetched) + self.search_result_from_cr.update(cr_result) + self.total = cr_result["message"]["total-results"] + self.app.logger.info("CROSSREF DONE") + + self.app.logger.info("CROSSREF DONE") + with open("cr_results.json", "r+", encoding="utf-8") as data_file: + # First we load existing data into a dict. + try: + file_data = json.load(data_file) + except JSONDecodeError: + file_data = {} + data_file.truncate(0) + data_file.seek(0) + tmp = {self.uuid : self.search_result_from_cr} + if file_data: + file_data.update(tmp) + else: + file_data = tmp + json.dump(file_data, data_file, indent=4) + return cr_result + + + async def refine_search(self, unrefined_result): + """ + Refines the search query based on the unrefined search result. + + Args: + - unrefined_result: The unrefined search result. + + Returns: + - refined_result: The refined search result. + + Raises: + - None. + """ + summarized_results = [] + self.app.logger.info("REFINE: Removing all derived works from the list") + for item in unrefined_result["message"]["items"]: + summarized_results.append( + { + "DOI": item["DOI"], + "title": item["title"] if "title" in item else None, + "type": item["type"] if "type" in item else None, + } + ) + + partial_result = { + self.uuid: { + "total_results": self.search_result_from_cr["message"][ + "total-results" + ], + "on_page": 1, + "summary": summarized_results, + "results": self.search_result_from_cr["message"]["items"], + } + } + self.app.logger.info("REFINE: Dumping to file") + temp = [] + refined_result = {} + for key in partial_result: + self.app.logger.info("KEY:") + self.app.logger.info(key) + + for item in partial_result[self.uuid]["summary"]: + if item["title"]: + temp.append(item) + + for item in temp: + refined_result[item["DOI"]]= item with open("rr_results.json", "r+", encoding="utf-8") as data_file: # First we load existing data into a dict. try: file_data = json.load(data_file) except JSONDecodeError: file_data = {} - summarized_results = [] - self.app.logger.info("REFINE: Removing all derived works from the list") - for item in self.search_result_from_cr["message"]["items"]: - container = "NO" - if "container-title" in item: - container = "YES" - else: - summarized_results.append( - { - "DOI": item["DOI"], - "title": item["title"] if "title" in item else None, - "type": item["type"] if "type" in item else None, - "container": container, - } - ) - result = { - self.uuid: { - "total_results": self.search_result_from_cr["message"][ - "total-results" - ], - "on_page": 1, - "summary": summarized_results, - "results": self.search_result_from_cr["message"]["items"], - } - } - if file_data: - file_data.update(result) - else: - file_data = result - - self.app.logger.info("REFINE: Dumping to file") data_file.truncate(0) data_file.seek(0) + tmp = {self.uuid: refined_result} + if file_data: + file_data.update(tmp) + else: + file_data = tmp json.dump(file_data, data_file, indent=4) - return result + return refined_result - async def check_dois_in_local_db(self, doi): - """ - Checks if the given DOI exists in the local database. - - Args: - - doi: The DOI to be checked. - - Returns: - - result: The search result from the backend app. - - Raises: - - None. - """ - self.app.logger.info("REFINE: Running search in the backend app") - coro = asyncio.to_thread( - search_bot.search_for_doi, doi, self.live_results, self.app.logger - ) - result = await coro - return result - - async def check_if_exists_brute_force(self, page_url): - """ - Checks if the given page URL exists using brute force method. - - Args: - - page_url: The URL of the page to be checked. - - Returns: - - exists: A boolean indicating if the page exists. - - Raises: - - ValueError: If the URL does not start with 'http:' or 'https:'. - """ - self.app.logger.error("REFINE: Brute force search!") - if not page_url.startswith(("http:", "https:")): - raise ValueError("URL must start with 'http:' or 'https:'") - # trunk-ignore(bandit/B310) - with urlopen(page_url) as response: - data = response.read() - text = data.decode("utf-8") - for line in text.splitlines(): - if re.match( - r".*Unfortunately, Sci-Hub doesn't have the requested document.*", - line, - ): - return False - if m := re.match( - r".* (doi, True) -def consumer(in_q, control_q, doi, live_results, logger): + + +# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK +def consumer(in_q, control_q, doi, live_results, result_list, _logger, ): + """ + Consumes items from an input queue and checks if DOI exists in the live results list. + + Args: + in_q (Queue): Input queue. + control_q (Queue): Control queue. + doi (list): List of DOI to search for. + live_results (list): List to store the search results. + _logger: Logger object for logging. + """ sentinels = 0 for item in doi: - result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]}) + result_list.append({"DOI": item[0], "exists": False, "data": item[1]}) while True: done_check = True data = in_q.get() if data is _sentinel: - logger.info("Worker finished") + _logger.info("Worker finished") sentinels += 1 else: for item in result_list: if item["DOI"] in data and not item["exists"]: - logger.info(data) - logger.info("HIT") + _logger.info(data) + _logger.info("HIT") item["exists"] = True live_results.append(item) done_check = done_check and item["exists"] if done_check: control_q.put(_sentinel) if sentinels >= MAXTHREADS: - logger.info("All workers finished") + _logger.info("All workers finished") break -def search_for_doi(doi, live_results, logger): +def search_for_doi(doi, live_results, _logger): + """ + Search for DOI in live_results using _logger for logging. + + Args: + doi (list): List of DOI to search for. + live_results (list): List to store the search results. + _logger: Logger object for logging. + """ + result_list = [] threads = [] work_q = Queue(maxsize=WORK_Q_SIZE) control_q = Queue() - t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger)) - logger.info("Consumer thread created") + t_cons = Thread( + target=consumer, args=(work_q, control_q, doi, live_results, result_list, _logger) + ) + _logger.info("Consumer thread created") threads.append(t_cons) - for i in range(0,MAXTHREADS): - #TODO: TEST DATA - #filename = "test" + str(i) + CHUNk - #TODO: DEPLOYMENT DATA - filename = str(i) + CHUNk - logger.info(f"Creating worker thread no: {i}") - threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger))) + for i in range(0, MAXTHREADS): + # TEST DATA + # filename = "test" + str(i) + CHUNk + # DEPLOYMENT DATA + filename = str(i) + CHUNK + _logger.info(f"Creating worker thread no: {i}") + threads.append( + Thread(target=producer, args=(work_q, control_q, filename, _logger)) + ) for worker in threads: worker.start() for worker in threads: worker.join() return result_list + if __name__ == "__main__": import logging + logger = logging.getLogger() logger.setLevel("DEBUG") h1 = logging.StreamHandler() logger.addHandler(h1) logger.info("TEST RUN") live_result = [] - logger.info(search_for_doi([ - ("10.1002/9781118786352.wbieg0998.pub2","DATA"), - ("10.1002/j.2050-0416.2002.tb00563.x","DATA"), - ("10.1111/j.1365-2958.1994.tb00448.x","DATA"), - ("10.2165/00128415-200309690-00017","DATA"), - ("10.5772/48313","DATA"), - ("10.15803/ijnc.7.2_419","DATA"), - ("10.1051/0004-6361/201321596e","DATA"), - ("10.2307/40835941","DATA"), - ], live_result, logger)) + logger.info( + search_for_doi( + [ + ("10.1002/9781118786352.wbieg0998.pub2", "DATA"), + ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"), + ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"), + ("10.2165/00128415-200309690-00017", "DATA"), + ("10.5772/48313", "DATA"), + ("10.15803/ijnc.7.2_419", "DATA"), + ("10.1051/0004-6361/201321596e", "DATA"), + ("10.2307/40835941", "DATA"), + ], + live_result, + logger, + ) + ) logger.info(live_result)