diff --git a/.trunk/.gitignore b/.trunk/.gitignore index 1e24652..15966d0 100644 --- a/.trunk/.gitignore +++ b/.trunk/.gitignore @@ -6,3 +6,4 @@ plugins user_trunk.yaml user.yaml +tmp diff --git a/.trunk/configs/.yamllint.yaml b/.trunk/configs/.yamllint.yaml index 4d44466..184e251 100644 --- a/.trunk/configs/.yamllint.yaml +++ b/.trunk/configs/.yamllint.yaml @@ -2,9 +2,6 @@ rules: quoted-strings: required: only-when-needed extra-allowed: ["{|}"] - empty-values: - forbid-in-block-mappings: true - forbid-in-flow-mappings: true key-duplicates: {} octal-values: forbid-implicit-octal: true diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 4473760..73b11de 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -1,43 +1,36 @@ +# This file controls the behavior of Trunk: https://docs.trunk.io/cli +# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml version: 0.1 cli: - version: 1.14.2 + version: 1.20.1 +# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) plugins: sources: - id: trunk - ref: v1.2.1 + ref: v1.4.4 uri: https://github.com/trunk-io/plugins -lint: - enabled: - - checkov@2.4.9 - - yamllint@1.32.0 - - actionlint@1.6.25 - - markdownlint@0.35.0 - - trivy@0.44.1 - - trufflehog@3.54.0 - - pylint - - codespell - - cspell - - git-diff-check - - eslint - - prettier - - semgrep - - autopep8 - - bandit - - black - - flake8 - - isort - - mypy - - pylint - - ruff - - semgrep - - yapf - - gitleaks - +# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) runtimes: enabled: - - go@1.19.5 - node@18.12.1 - python@3.10.8 +# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) +lint: + enabled: + - actionlint@1.6.27 + - bandit@1.7.7 + - black@24.2.0 + - codespell + - checkov@3.2.32 + - git-diff-check + - isort@5.13.2 + - pylint + - markdownlint@0.39.0 + - prettier@3.2.5 + - ruff@0.3.1 + - trivy@0.49.1 + - trufflehog@3.68.5 + - yamllint@1.35.1 actions: disabled: - trunk-announce diff --git a/.vscode/settings.json b/.vscode/settings.json index a5026fb..f798400 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,4 @@ { - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, "trunk.inlineDecorators": true, "trunk.inlineDecoratorsForAllExtensions": true, "trunk.showPreexistingIssues": true, diff --git a/bot.py b/bot.py index 837173a..cfef22a 100644 --- a/bot.py +++ b/bot.py @@ -1,12 +1,10 @@ +# This Python file uses the following encoding: utf-8 +# trunk-ignore-all(bandit/B311) +# pylint: disable=line-too-long +# pylint: disable=too-many-lines """ Module of a python bot named Conjurer - used to work on BDSM discord servers. """ -# This Python file uses the following encoding: utf-8 -# trunk-ignore-all(flake8/E501) -# trunk-ignore-all(bandit/B311) -# trunk-ignore-all(mypy/index) -# pylint: disable=line-too-long -# pylint: disable=too-many-lines import asyncio import io import json @@ -40,8 +38,8 @@ from discord.ext import commands from spotipy.oauth2 import SpotifyClientCredentials import yt_dlp -from spotify_dl import youtube as youtube_download from spotify_dl import spotify +from spotify_dl import youtube as youtube_download Music_Config = TypedDict( "Music_Config", @@ -54,12 +52,10 @@ Music_Config = TypedDict( ) # *=========================================== Predefines -formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s") +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") MASTER_TIMEOUT = datetime.now() INITIAL_TIME_WAIT = 5 -MUZYKA: Music_Config = {"playing": False, - "ctx": None, "queue": [], "requester": []} +MUZYKA: Music_Config = {"playing": False, "ctx": None, "queue": [], "requester": []} LOGFILE = "" NETRC_FILE = "" @@ -209,7 +205,6 @@ async def on_ready(): # trunk-ignore(pylint/R0914) # trunk-ignore(pylint/R0912) async def on_message(message): - """ Handle incoming messages in a Discord server, perform various checks and actions based on the content and context of the message, and respond accordingly. @@ -342,6 +337,7 @@ async def on_message(message): # *=========================================== Define Functions + def num_tokens_from_string(message, model): """ The function takes a string message and a model as input and returns the number of tokens in the @@ -496,11 +492,16 @@ async def play(ctx, zamawial=None, arg=None): async def archive_channel(channel_no): + """ + The function `archive_channel` retrieves messages from a specified channel, processes them, and + saves the data in a JSON file with a timestamp in the filename. + + :param channel_no: The `archive_channel` function you provided seems to be incomplete. It looks like + you are trying to archive messages from a Discord channel into a JSON file + """ channel = client.get_channel(channel_no) messages = [message async for message in channel.history(limit=None)] - path_newfile = ( - f"{LOGSTORE}channeldump_{channel_no}_{datetime.now()}.json" - ) + path_newfile = f"{LOGSTORE}channeldump_{channel_no}_{datetime.now()}.json" new_data = [] for message in messages: pass @@ -594,7 +595,7 @@ async def get_random_cyclic_message(): words. :return: a random cyclic message from the list `cyclic_words`. """ - messnum = random.randint(0, len(cyclic_words)-1) + messnum = random.randint(0, len(cyclic_words) - 1) return cyclic_words[messnum][0] @@ -646,15 +647,22 @@ async def handle_response(prompt, vykidailo, bartender, history, username, music history.append(GPT_SETTINGS[0]) chat_gpt_config_request_size = num_tokens_from_string(GPT_SETTINGS[0], "gpt-4") - for (slowo, reakcja) in word_reactions.items(): + for slowo, reakcja in word_reactions.items(): if not reakcja[3]: - content = "Kiedy słyszysz " + slowo + " to reagujesz lub dzieje się to " + reakcja[0] + content = ( + "Kiedy słyszysz " + + slowo + + " to reagujesz lub dzieje się to " + + reakcja[0] + ) temp = {"role": "system", "content": content} chat_gpt_config_request_size += num_tokens_from_string(temp, "gpt-4") history.append(temp) final_prompt = username + ":" + prompt - logger.info("Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size) + logger.info( + "Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size + ) if music: algorithm = "gpt-3.5-turbo-0125" table = message_table_muzyka @@ -664,12 +672,22 @@ async def handle_response(prompt, vykidailo, bartender, history, username, music algorithm = "gpt-4" token_amount = 7000 - prompt_gpt_request_size = num_tokens_from_string({"role": "user", "content": final_prompt}, "gpt-4") + prompt_gpt_request_size = num_tokens_from_string( + {"role": "user", "content": final_prompt}, "gpt-4" + ) temptable = [] for i in reversed(table): temp_token = num_tokens_from_string(i, "gpt-4") - logger.info("Rozmiar zapytania %s prompt %s temp %s", chat_gpt_config_request_size, prompt_gpt_request_size, temp_token) - if chat_gpt_config_request_size < token_amount + prompt_gpt_request_size + temp_token: + logger.info( + "Rozmiar zapytania %s prompt %s temp %s", + chat_gpt_config_request_size, + prompt_gpt_request_size, + temp_token, + ) + if ( + chat_gpt_config_request_size + < token_amount + prompt_gpt_request_size + temp_token + ): temptable.insert(1, i) chat_gpt_config_request_size += temp_token history.extend(temptable) @@ -677,8 +695,8 @@ async def handle_response(prompt, vykidailo, bartender, history, username, music history.append(temp) logger.info("Rozmiar zapytania po wyslaniu %s", chat_gpt_config_request_size) response = await openAIClient.chat.completions.create( - model=algorithm, - messages=history) + model=algorithm, messages=history + ) logger.info("Historia wysłana:") logger.info(history) @@ -741,8 +759,7 @@ async def get_file(ctx, source, link): item_type, item_id = spotify.parse_spotify_url(link) if item_id: file_list = spotify.fetch_tracks(spotify_ctrl, item_type, link) - directory_name = spotify.get_item_name( - spotify_ctrl, item_type, item_id) + directory_name = spotify.get_item_name(spotify_ctrl, item_type, item_id) url_data = {"urls": []} url_dict = {} url_dict["save_path"] = Path( @@ -977,15 +994,13 @@ async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None): if re.match(pattern, itm, re.IGNORECASE): temp_weight = ( search_weight[itr][0] - + (token_weight + (pingu**1.5) / - pattern_len) / matched_times + + (token_weight + (pingu**1.5) / pattern_len) / matched_times ) search_weight[itr] = temp_weight, music_file_list[itr] matched_times += 1 itr += 1 - logger.info("Stworzylem tablice wag zajęło mi to %s", - datetime.now() - time_start) + logger.info("Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start) time_start = datetime.now() item_to_search = await max_weight(search_weight) itr = 0 @@ -1038,7 +1053,6 @@ async def max_weight(lista): # *=========================================== Define Commands -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="dej_co_ze_spotifaja", description="Podaj link do spotify - sciagnie i doda muzyke", @@ -1084,7 +1098,6 @@ async def dej_co_ze_spotifaja(ctx): logger.info("Spotifaj udany") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="dej_co_z_jutuba", description="Podaj link do youtube - sciagnie i doda muzyke", @@ -1117,7 +1130,6 @@ async def dej_co_z_jutuba(ctx): logger.info("Jutub udany") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="update_banlist", description="Jeśli nie wiesz jak użyć tej komendy to nawet nie próbuj", @@ -1157,7 +1169,7 @@ async def update_banlist(ctx): for line in lines: match = re.search( "\\d{18}", - line + line, # trunk-ignore(codespell/misspelled) ) # poprawilem backslash na podwojny bo linter sie czepial if match: @@ -1208,7 +1220,6 @@ async def update_banlist(ctx): await ctx.send("Zrobione tak czy inaczej") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( nsfw=True, name="get_image_sadox", @@ -1253,7 +1264,6 @@ async def get_image_sadox(ctx): logger.info("Get sadox completed") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="graj_muzyko", description="Włącza muzykę na kanale #nocna-zmiana.", @@ -1279,7 +1289,6 @@ async def graj_muzyko(ctx): logger.info("Press play on tape completed") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="cisza", description="Wyłącza muzykę i czyści plejliste.", @@ -1307,7 +1316,6 @@ async def cisza(ctx): logger.info("Stop completed") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="dalej", description="Przerzuca na następny kawałek", @@ -1332,7 +1340,6 @@ async def dalej(ctx): logger.info("Next completed") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="daj_mi_chwile", description="Pauzuje", @@ -1357,7 +1364,6 @@ async def daj_mi_chwile(ctx): logger.info("Pause completed") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="graj_dalej", description="Odpauzowuje", @@ -1381,7 +1387,6 @@ async def graj_dalej(ctx): logger.info("Unpause completed") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="zagraj_mi_kawalek", description="Wyszukuje w bibliotece muzycznej Hammera kawałek który ma zostać zagrany", @@ -1428,7 +1433,6 @@ async def zagraj_mi_kawalek(ctx): ) -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="zrob_mi_plejliste", description="Generuje playliste - param1 - dlugosc, potem slowa wyszukiwania", @@ -1490,7 +1494,6 @@ async def zrob_mi_plejliste(ctx): logger.info("Plejlista zrobiona") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="przytul", description="Przytul kogoś - daj mention po komendzie :)" ) @@ -1527,7 +1530,6 @@ async def przytul(ctx, arg: Optional[discord.Member] = None): ) -# trunk-ignore(mypy/arg-type) @client.hybrid_command(name="fabryczka", description="Historia fabryczki") async def fabryczka(ctx): """ @@ -1541,7 +1543,6 @@ async def fabryczka(ctx): await ctx.send(historia_fabryczki) -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="chata_hammera", description="Czas od ostatniego incydentu w AbsinthHammerTimeSpaceContinuum", @@ -1565,7 +1566,6 @@ async def chata_hammera(ctx): ) -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="historia_incydentow_u_hammera", description="Wyswietla liste incydentów które miały miejsce w AbsinthHammerTimeSpaceContinuum.", @@ -1593,7 +1593,6 @@ async def historia_incydentow_u_hammera(ctx): await ctx.send(return_data) -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="reset_the_clock", description="Resetowanie zegara - dostępne wyłącznie dla Hammera", @@ -1646,7 +1645,6 @@ async def get_image_stable_diffusion(ctx): pass -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="zagraj_muzyke_mego_ludu", description="Gra muzyke wyszukana na bazie tematow konwersacji na kanale NZ", @@ -1687,7 +1685,11 @@ async def zagraj_muzyke_mego_ludu(ctx): async with ctx.typing(): logger.info("Zaczynam szukać timestamp %s", datetime.now()) search_time_glob = datetime.now() - file = await wyszukaj(ctx=ctx, how_many=MUZYKA_MOJEGO_LUDU_PLAJLISTA, slowa_kluczowe=final_key_words) + file = await wyszukaj( + ctx=ctx, + how_many=MUZYKA_MOJEGO_LUDU_PLAJLISTA, + slowa_kluczowe=final_key_words, + ) logger.info("Zagraj muzyke mego ludu:wygenerowano plejliste") logger.info(file) logger.info( @@ -1729,13 +1731,14 @@ async def zagraj_muzyke_mego_ludu(ctx): logger.info("Plejlista zrobiona") -# trunk-ignore(mypy/arg-type) @client.hybrid_command( name="parametry_muzyki_mego_ludu", description="Konfiguruje komende zagraj muzyke mojego ludu", guild=discord.Object(id=664789470779932693), ) -async def parametry_muzyki_mego_ludu(ctx, ile_historii=1500, ile_slow_kluczowych=15, jak_dluga_plejlista=30): +async def parametry_muzyki_mego_ludu( + ctx, ile_historii=1500, ile_slow_kluczowych=15, jak_dluga_plejlista=30 +): """ The function `parametry_muzyki_mego_ludu` sets global variables for the number of history entries, number of keywords, and length of playlist for a music application. @@ -1764,18 +1767,31 @@ async def parametry_muzyki_mego_ludu(ctx, ile_historii=1500, ile_slow_kluczowych global MUZYKA_MOJEGO_LUDU_HISTORIA # pylint: disable=global-statement global MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE # pylint: disable=global-statement global MUZYKA_MOJEGO_LUDU_PLAJLISTA # pylint: disable=global-statement - logger.info("Długość historii wyszukiwania: %s\n Ilość słów kluczowych: %s\n Długość plejlisty do wygenerowanie: %s", MUZYKA_MOJEGO_LUDU_HISTORIA, MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE, MUZYKA_MOJEGO_LUDU_PLAJLISTA) - ctx.send(f"Dotychczasowe wartośći:\n Długość historii wyszukiwania: {MUZYKA_MOJEGO_LUDU_HISTORIA}\n Ilość słów kluczowych: {MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE}\n Długość plejlisty do wygenerowanie:{MUZYKA_MOJEGO_LUDU_PLAJLISTA}") + logger.info( + "Długość historii wyszukiwania: %s\n Ilość słów kluczowych: %s\n Długość plejlisty do wygenerowanie: %s", + MUZYKA_MOJEGO_LUDU_HISTORIA, + MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE, + MUZYKA_MOJEGO_LUDU_PLAJLISTA, + ) + ctx.send( + f"Dotychczasowe wartośći:\n Długość historii wyszukiwania: {MUZYKA_MOJEGO_LUDU_HISTORIA}\n Ilość słów kluczowych: {MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE}\n Długość plejlisty do wygenerowanie:{MUZYKA_MOJEGO_LUDU_PLAJLISTA}" + ) MUZYKA_MOJEGO_LUDU_HISTORIA = ile_historii MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = ile_slow_kluczowych MUZYKA_MOJEGO_LUDU_PLAJLISTA = jak_dluga_plejlista - logger.info("Długość historii wyszukiwania: %s\n Ilość słów kluczowych: %s\n Długość plejlisty do wygenerowanie: %s", MUZYKA_MOJEGO_LUDU_HISTORIA, MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE, MUZYKA_MOJEGO_LUDU_PLAJLISTA) - ctx.send(f"Zmienione wartośći:\n Długość historii wyszukiwania: {MUZYKA_MOJEGO_LUDU_HISTORIA}\n Ilość słów kluczowych: {MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE}\n Długość plejlisty do wygenerowanie:{MUZYKA_MOJEGO_LUDU_PLAJLISTA}") + logger.info( + "Długość historii wyszukiwania: %s\n Ilość słów kluczowych: %s\n Długość plejlisty do wygenerowanie: %s", + MUZYKA_MOJEGO_LUDU_HISTORIA, + MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE, + MUZYKA_MOJEGO_LUDU_PLAJLISTA, + ) + ctx.send( + f"Zmienione wartośći:\n Długość historii wyszukiwania: {MUZYKA_MOJEGO_LUDU_HISTORIA}\n Ilość słów kluczowych: {MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE}\n Długość plejlisty do wygenerowanie:{MUZYKA_MOJEGO_LUDU_PLAJLISTA}" + ) except discord.ext.commands.errors.BadArgument as exce: ctx.send("Spierdalaj") logger.info(exce) # *================================== Run -# trunk-ignore(mypy/arg-type) client.run(TOKEN) diff --git a/cut_scihub_database.py b/cut_scihub_database.py index 1316f12..45b772f 100644 --- a/cut_scihub_database.py +++ b/cut_scihub_database.py @@ -1,188 +1,58 @@ -import re -import time -output_size = 0 -preface = True -header_no = 0 -output_no = 0 -header = False -body = False -footer = False -file_limit = 500000*1024 +# trunk-ignore-all(pylint/C0301) + +FILE_SIZE_LIMIT = 100000 * 1024 +SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql" +OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/" +OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql" -def smart_write(line, file): - encoded_bytes = line.encode("utf-8") +def smart_write(lines, file): + """ + The function `smart_write` encodes the input lines to UTF-8, writes them to a file, and returns the + size of the encoded bytes. + + :param lines: The `lines` parameter in the `smart_write` function is a string that contains the text + content that you want to write to a file + :param file: The `file` parameter in the `smart_write` function is expected to be a file object that + has been opened in write mode. This file object will be used to write the encoded `lines` to a file + :return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in + bytes. + """ + encoded_bytes = lines.encode("utf-8") size_in_bytes = len(encoded_bytes) - op_file.write(line) + file.write(lines) + return size_in_bytes -with open( - "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql", encoding="utf-8" -) as infile: - for line in infile: - line.replace("`", "'") +def cut_my_life_into_pieces(source, output_folder, output_file): + """ + The function `cut_my_life_into_pieces` reads a source file, splits its content into smaller parts + based on a file size limit, and writes these parts to separate output files in a specified output + folder. - if re.compile("INSERT").search(line): - if preface: - preface = False - body = True - op_file_name = "scimag_chunk_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) + :param source: The `source` parameter in the `cut_my_life_into_pieces` function is the file path of + the source file that you want to process and split into smaller pieces + :param output_folder: The `output_folder` parameter in the `cut_my_life_into_pieces` function + represents the directory where the output files will be saved. It should be a string that specifies + the path to the folder where you want to store the output files generated by the function. For + example, it could be something like + :param output_file: The `output_file` parameter in the `cut_my_life_into_pieces` function represents + the name of the output file that will be created for each segment of the source file. It will be + appended with a number to differentiate between different segments + """ + with open(source, encoding="utf-8") as infile: + output_size = 0 + output_no = 0 - elif header: - header = False - body = True - op_file_name = "scimag_chunk_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - - elif footer: - print("ERROR: Insert after footer (without header)") - elif body: - if output_size > file_limit: - output_no += 1 - output_size = 0 - op_file_name = "scimag_chunk_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - else: - print("ERROR: Insert with no state") - elif re.compile("UNLOCK").search(line): - if preface: - print("ERROR: UNLOCK in preface state") - - elif header: - header = False - footer = True - op_file_name = "scimag_chunk_footer{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - elif footer: - op_file_name = "scimag_chunk_footer{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - elif body: - body = False - footer = True - op_file_name = "scimag_chunk_footer{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - - else: - print("ERROR: Unlock in unknown state") - - elif re.compile("LOCK").search(line): - header_no += 1 - if preface: - preface = False - header = True - op_file_name = "scimag_chunk_header_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - - elif header: - op_file_name = "scimag_chunk_header_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - - elif footer: - footer = False - header = True - op_file_name = "scimag_chunk_header_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - elif body: - body = False - header = True - op_file_name = "scimag_chunk_header_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - else: - print("ERROR: Lock in unknown state") - - else: - if preface: - op_file_name = "scimag_chunk_{}_preface.sql".format( - header_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - op_file.write(line) - elif header: - op_file_name = "scimag_chunk_header_{}_{}.sql".format( - header_no, output_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - output_size += smart_write(line, op_file) - elif body: - print("ERROR: Other command in locked state") - elif footer: - header_no += 1 - op_file_name = "scimag_chunk_{}_preface.sql".format( - header_no) - with open( - "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), - "a", - encoding="utf-8", - ) as op_file: - op_file.write(line) - - else: - print("ERROR: Other command in unknown state") \ No newline at end of file + for line in infile: + if output_size > FILE_SIZE_LIMIT: + output_no += 1 + output_size = 0 + op_file_name = str(output_no) + output_file + with open( + output_folder + op_file_name, + "a", + encoding="utf-8", + ) as op_file: + output_size += smart_write(line, op_file) diff --git a/import_to_mysql.py b/import_to_mysql.py index d7f13db..2bb84dc 100644 --- a/import_to_mysql.py +++ b/import_to_mysql.py @@ -1,3 +1,7 @@ +""" +Kluczowe jest zeby po każdej linijce pokazywał która to jest, oraz po pliku +""" + import netrc import mysql.connector @@ -17,7 +21,7 @@ mycursor.execute("SELECT * FROM scimag") for x in mycursor: print(x) -exit() + file_list = "" for file in file_list: diff --git a/import_to_postgres.py b/import_to_postgres.py deleted file mode 100644 index bab791c..0000000 --- a/import_to_postgres.py +++ /dev/null @@ -1,33 +0,0 @@ -import netrc - -import psycopg2 - -NETRC_FILE = "" -NETRC_FILE = "/home/mtuszowski/.netrc" -# NETRC_FILE = "/home/pi/.netrc" -# NETRC_FILE = "C:\\Users\\mtusz\\.netrc" - -netrc_mod = netrc.netrc(NETRC_FILE) -REMOTE_HOST_NAME = "sql_localhost" -authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME) - -# Connect to your postgres DB -conn = psycopg2.connect( - database="scihubdois", - host="localhost", - # trunk-ignore(mypy/index) - user=authTokens[0], - # trunk-ignore(mypy/index) - password=authTokens[2], - port="5432", -) - -# Open a cursor to perform database operations -cur = conn.cursor() - -# Execute a query -cur.execute("SELECT * FROM version()") - -# Retrieve query results -records = cur.fetchall() -print(records) diff --git a/test_data_retrieval.py b/test_data_retrieval.py new file mode 100644 index 0000000..32dd665 --- /dev/null +++ b/test_data_retrieval.py @@ -0,0 +1,26 @@ +""" +This Python code snippet is connecting to a MySQL database +using the `mysql.connector` library and fetching data from a table named `scimag` +Here is a breakdown of what the code is doing: +""" + +import netrc + +import mysql.connector + +NETRC_FILE = "/home/mtuszowski/.netrc" +netrc_mod = netrc.netrc(NETRC_FILE) +REMOTE_HOST_NAME = "192.168.1.192" +authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME) + +mydb = mysql.connector.connect( + host=REMOTE_HOST_NAME, + user=authTokens[0], + password=authTokens[2], + database="scihubdois", +) +mycursor = mydb.cursor() + +mycursor.execute("SELECT * FROM scimag WHERE Title Like 'spider%'") +for x in mycursor: + print(x)