mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
Nowy lepszy trunk
This commit is contained in:
+50
-180
@@ -1,188 +1,58 @@
|
||||
import re
|
||||
import time
|
||||
output_size = 0
|
||||
preface = True
|
||||
header_no = 0
|
||||
output_no = 0
|
||||
header = False
|
||||
body = False
|
||||
footer = False
|
||||
file_limit = 500000*1024
|
||||
# trunk-ignore-all(pylint/C0301)
|
||||
|
||||
FILE_SIZE_LIMIT = 100000 * 1024
|
||||
SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
|
||||
OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
|
||||
OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
|
||||
|
||||
|
||||
def smart_write(line, file):
|
||||
encoded_bytes = line.encode("utf-8")
|
||||
def smart_write(lines, file):
|
||||
"""
|
||||
The function `smart_write` encodes the input lines to UTF-8, writes them to a file, and returns the
|
||||
size of the encoded bytes.
|
||||
|
||||
:param lines: The `lines` parameter in the `smart_write` function is a string that contains the text
|
||||
content that you want to write to a file
|
||||
:param file: The `file` parameter in the `smart_write` function is expected to be a file object that
|
||||
has been opened in write mode. This file object will be used to write the encoded `lines` to a file
|
||||
:return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
|
||||
bytes.
|
||||
"""
|
||||
encoded_bytes = lines.encode("utf-8")
|
||||
size_in_bytes = len(encoded_bytes)
|
||||
op_file.write(line)
|
||||
file.write(lines)
|
||||
|
||||
return size_in_bytes
|
||||
|
||||
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql", encoding="utf-8"
|
||||
) as infile:
|
||||
for line in infile:
|
||||
line.replace("`", "'")
|
||||
def cut_my_life_into_pieces(source, output_folder, output_file):
|
||||
"""
|
||||
The function `cut_my_life_into_pieces` reads a source file, splits its content into smaller parts
|
||||
based on a file size limit, and writes these parts to separate output files in a specified output
|
||||
folder.
|
||||
|
||||
if re.compile("INSERT").search(line):
|
||||
if preface:
|
||||
preface = False
|
||||
body = True
|
||||
op_file_name = "scimag_chunk_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
:param source: The `source` parameter in the `cut_my_life_into_pieces` function is the file path of
|
||||
the source file that you want to process and split into smaller pieces
|
||||
:param output_folder: The `output_folder` parameter in the `cut_my_life_into_pieces` function
|
||||
represents the directory where the output files will be saved. It should be a string that specifies
|
||||
the path to the folder where you want to store the output files generated by the function. For
|
||||
example, it could be something like
|
||||
:param output_file: The `output_file` parameter in the `cut_my_life_into_pieces` function represents
|
||||
the name of the output file that will be created for each segment of the source file. It will be
|
||||
appended with a number to differentiate between different segments
|
||||
"""
|
||||
with open(source, encoding="utf-8") as infile:
|
||||
output_size = 0
|
||||
output_no = 0
|
||||
|
||||
elif header:
|
||||
header = False
|
||||
body = True
|
||||
op_file_name = "scimag_chunk_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
|
||||
elif footer:
|
||||
print("ERROR: Insert after footer (without header)")
|
||||
elif body:
|
||||
if output_size > file_limit:
|
||||
output_no += 1
|
||||
output_size = 0
|
||||
op_file_name = "scimag_chunk_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
else:
|
||||
print("ERROR: Insert with no state")
|
||||
elif re.compile("UNLOCK").search(line):
|
||||
if preface:
|
||||
print("ERROR: UNLOCK in preface state")
|
||||
|
||||
elif header:
|
||||
header = False
|
||||
footer = True
|
||||
op_file_name = "scimag_chunk_footer{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
elif footer:
|
||||
op_file_name = "scimag_chunk_footer{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
elif body:
|
||||
body = False
|
||||
footer = True
|
||||
op_file_name = "scimag_chunk_footer{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
|
||||
else:
|
||||
print("ERROR: Unlock in unknown state")
|
||||
|
||||
elif re.compile("LOCK").search(line):
|
||||
header_no += 1
|
||||
if preface:
|
||||
preface = False
|
||||
header = True
|
||||
op_file_name = "scimag_chunk_header_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
|
||||
elif header:
|
||||
op_file_name = "scimag_chunk_header_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
|
||||
elif footer:
|
||||
footer = False
|
||||
header = True
|
||||
op_file_name = "scimag_chunk_header_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
elif body:
|
||||
body = False
|
||||
header = True
|
||||
op_file_name = "scimag_chunk_header_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
else:
|
||||
print("ERROR: Lock in unknown state")
|
||||
|
||||
else:
|
||||
if preface:
|
||||
op_file_name = "scimag_chunk_{}_preface.sql".format(
|
||||
header_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
op_file.write(line)
|
||||
elif header:
|
||||
op_file_name = "scimag_chunk_header_{}_{}.sql".format(
|
||||
header_no, output_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
elif body:
|
||||
print("ERROR: Other command in locked state")
|
||||
elif footer:
|
||||
header_no += 1
|
||||
op_file_name = "scimag_chunk_{}_preface.sql".format(
|
||||
header_no)
|
||||
with open(
|
||||
"/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
op_file.write(line)
|
||||
|
||||
else:
|
||||
print("ERROR: Other command in unknown state")
|
||||
for line in infile:
|
||||
if output_size > FILE_SIZE_LIMIT:
|
||||
output_no += 1
|
||||
output_size = 0
|
||||
op_file_name = str(output_no) + output_file
|
||||
with open(
|
||||
output_folder + op_file_name,
|
||||
"a",
|
||||
encoding="utf-8",
|
||||
) as op_file:
|
||||
output_size += smart_write(line, op_file)
|
||||
|
||||
Reference in New Issue
Block a user