import re import time output_size = 0 preface = True header_no = 0 output_no = 0 header = False body = False footer = False file_limit = 500000*1024 def smart_write(line, file): encoded_bytes = line.encode("utf-8") size_in_bytes = len(encoded_bytes) op_file.write(line) return size_in_bytes with open( "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql", encoding="utf-8" ) as infile: for line in infile: line.replace("`", "'") if re.compile("INSERT").search(line): if preface: preface = False body = True op_file_name = "scimag_chunk_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif header: header = False body = True op_file_name = "scimag_chunk_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif footer: print("ERROR: Insert after footer (without header)") elif body: if output_size > file_limit: output_no += 1 output_size = 0 op_file_name = "scimag_chunk_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) else: print("ERROR: Insert with no state") elif re.compile("UNLOCK").search(line): if preface: print("ERROR: UNLOCK in preface state") elif header: header = False footer = True op_file_name = "scimag_chunk_footer{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif footer: op_file_name = "scimag_chunk_footer{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif body: body = False footer = True op_file_name = "scimag_chunk_footer{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) else: print("ERROR: Unlock in unknown state") elif re.compile("LOCK").search(line): header_no += 1 if preface: preface = False header = True op_file_name = "scimag_chunk_header_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif header: op_file_name = "scimag_chunk_header_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif footer: footer = False header = True op_file_name = "scimag_chunk_header_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif body: body = False header = True op_file_name = "scimag_chunk_header_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) else: print("ERROR: Lock in unknown state") else: if preface: op_file_name = "scimag_chunk_{}_preface.sql".format( header_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: op_file.write(line) elif header: op_file_name = "scimag_chunk_header_{}_{}.sql".format( header_no, output_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) elif body: print("ERROR: Other command in locked state") elif footer: header_no += 1 op_file_name = "scimag_chunk_{}_preface.sql".format( header_no) with open( "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name), "a", encoding="utf-8", ) as op_file: op_file.write(line) else: print("ERROR: Other command in unknown state")