Move

utils/cut_large_file.py

@@ -0,0 +1,73 @@
+# trunk-ignore-all(pylint/C0301)
+"""
+The function `cut_my_life_into_pieces` reads a source file, splits its content based on a file size
+limit, and writes the parts to separate output files in a specified folder.
+
+:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
+content that you want to write to a file. It represents the actual content that will be written to
+the file
+:param file: The code you provided defines two functions: `smart_write` and
+`cut_my_life_into_pieces`. The `smart_write` function encodes input lines to UTF-8, writes them to a
+file, and returns the size of the encoded bytes. The `cut_my_life_into_pieces` function reads
+:return: The functions provided in the code snippet do not explicitly return any values. However,
+the `smart_write` function returns the size of the encoded bytes of the input `lines` in bytes. The
+`cut_my_life_into_pieces` function does not have a return statement, so it does not explicitly
+return any value.
+"""
+
+FILE_SIZE_LIMIT = 100000 * 1024
+SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
+OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
+OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
+
+
+def smart_write(lines, file):
+    """
+    The function `smart_write` encodes the input lines to UTF-8, writes them to a file, and returns the
+    size of the encoded bytes.
+
+    :param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
+    content that you want to write to a file
+    :param file: The `file` parameter in the `smart_write` function is expected to be a file object that
+    has been opened in write mode. This file object will be used to write the encoded `lines` to a file
+    :return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
+    bytes.
+    """
+    encoded_bytes = lines.encode("utf-8")
+    size_in_bytes = len(encoded_bytes)
+    file.write(lines)
+
+    return size_in_bytes
+
+
+def cut_my_life_into_pieces(source, output_folder, output_file):
+    """
+    The function `cut_my_life_into_pieces` reads a source file, splits its content into smaller parts
+    based on a file size limit, and writes these parts to separate output files in a specified output
+    folder.
+
+    :param source: The `source` parameter in the `cut_my_life_into_pieces` function is the file path of
+    the source file that you want to process and split into smaller pieces
+    :param output_folder: The `output_folder` parameter in the `cut_my_life_into_pieces` function
+    represents the directory where the output files will be saved. It should be a string that specifies
+    the path to the folder where you want to store the output files generated by the function. For
+    example, it could be something like
+    :param output_file: The `output_file` parameter in the `cut_my_life_into_pieces` function represents
+    the name of the output file that will be created for each segment of the source file. It will be
+    appended with a number to differentiate between different segments
+    """
+    with open(source, encoding="utf-8") as infile:
+        output_size = 0
+        output_no = 0
+
+        for line in infile:
+            if output_size > FILE_SIZE_LIMIT:
+                output_no += 1
+                output_size = 0
+            op_file_name = str(output_no) + output_file
+            with open(
+                output_folder + op_file_name,
+                "a",
+                encoding="utf-8",
+            ) as op_file:
+                output_size += smart_write(line, op_file)

utils/import_to_mysql.py

@@ -0,0 +1,33 @@
+"""
+Kluczowe jest zeby po każdej linijce pokazywał która to jest, oraz po pliku
+"""
+
+import netrc
+
+import mysql.connector
+
+NETRC_FILE = "/home/mtuszowski/.netrc"
+
+netrc_mod = netrc.netrc(NETRC_FILE)
+REMOTE_HOST_NAME = "sql_localhost"
+authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME)
+
+mydb = mysql.connector.connect(
+    host="localhost", user="root", password="root", database="scihubdois"
+)
+mycursor = mydb.cursor()
+
+mycursor.execute("SELECT * FROM scimag")
+for x in mycursor:
+    print(x)
+
+
+file_list = ""
+for file in file_list:
+
+    with open(
+        "/mnt/n/scimag_2020-05-30.sql/{}".format(file), encoding="utf-8"
+    ) as infile:
+        pass
+result = mycursor.execute("SHOW databases")
+# trunk-ignore(mypy/union-attr)