mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
59 lines
2.5 KiB
Python
59 lines
2.5 KiB
Python
# trunk-ignore-all(pylint/C0301)
|
|
|
|
FILE_SIZE_LIMIT = 100000 * 1024
|
|
SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
|
|
OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
|
|
OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
|
|
|
|
|
|
def smart_write(lines, file):
|
|
"""
|
|
The function `smart_write` encodes the input lines to UTF-8, writes them to a file, and returns the
|
|
size of the encoded bytes.
|
|
|
|
:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
|
|
content that you want to write to a file
|
|
:param file: The `file` parameter in the `smart_write` function is expected to be a file object that
|
|
has been opened in write mode. This file object will be used to write the encoded `lines` to a file
|
|
:return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
|
|
bytes.
|
|
"""
|
|
encoded_bytes = lines.encode("utf-8")
|
|
size_in_bytes = len(encoded_bytes)
|
|
file.write(lines)
|
|
|
|
return size_in_bytes
|
|
|
|
|
|
def cut_my_life_into_pieces(source, output_folder, output_file):
|
|
"""
|
|
The function `cut_my_life_into_pieces` reads a source file, splits its content into smaller parts
|
|
based on a file size limit, and writes these parts to separate output files in a specified output
|
|
folder.
|
|
|
|
:param source: The `source` parameter in the `cut_my_life_into_pieces` function is the file path of
|
|
the source file that you want to process and split into smaller pieces
|
|
:param output_folder: The `output_folder` parameter in the `cut_my_life_into_pieces` function
|
|
represents the directory where the output files will be saved. It should be a string that specifies
|
|
the path to the folder where you want to store the output files generated by the function. For
|
|
example, it could be something like
|
|
:param output_file: The `output_file` parameter in the `cut_my_life_into_pieces` function represents
|
|
the name of the output file that will be created for each segment of the source file. It will be
|
|
appended with a number to differentiate between different segments
|
|
"""
|
|
with open(source, encoding="utf-8") as infile:
|
|
output_size = 0
|
|
output_no = 0
|
|
|
|
for line in infile:
|
|
if output_size > FILE_SIZE_LIMIT:
|
|
output_no += 1
|
|
output_size = 0
|
|
op_file_name = str(output_no) + output_file
|
|
with open(
|
|
output_folder + op_file_name,
|
|
"a",
|
|
encoding="utf-8",
|
|
) as op_file:
|
|
output_size += smart_write(line, op_file)
|