conjurer/cut_scihub_database.py

import re
import time
output_size = 0
preface = True
header_no = 0
output_no = 0
header = False
body = False
footer = False
file_limit = 500000*1024


def smart_write(line, file):
    encoded_bytes = line.encode("utf-8")
    size_in_bytes = len(encoded_bytes)
    op_file.write(line)
    return size_in_bytes


with open(
    "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql", encoding="utf-8"
) as infile:
    for line in infile:
        line.replace("`", "'")

        if re.compile("INSERT").search(line):
            if preface:
                preface = False
                body = True
                op_file_name = "scimag_chunk_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)

            elif header:
                header = False
                body = True
                op_file_name = "scimag_chunk_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)

            elif footer:
                print("ERROR: Insert after footer (without header)")
            elif body:
                if output_size > file_limit:
                    output_no += 1
                    output_size = 0
                op_file_name = "scimag_chunk_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)
            else:
                print("ERROR: Insert with no state")
        elif re.compile("UNLOCK").search(line):
            if preface:
                print("ERROR: UNLOCK in preface state")

            elif header:
                header = False
                footer = True
                op_file_name = "scimag_chunk_footer{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)
            elif footer:
                op_file_name = "scimag_chunk_footer{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)
            elif body:
                body = False
                footer = True
                op_file_name = "scimag_chunk_footer{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)

            else:
                print("ERROR: Unlock in unknown state")

        elif re.compile("LOCK").search(line):
            header_no += 1
            if preface:
                preface = False
                header = True
                op_file_name = "scimag_chunk_header_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)

            elif header:
                op_file_name = "scimag_chunk_header_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)

            elif footer:
                footer = False
                header = True
                op_file_name = "scimag_chunk_header_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)
            elif body:
                body = False
                header = True
                op_file_name = "scimag_chunk_header_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)
            else:
                print("ERROR: Lock in unknown state")

        else:
            if preface:
                op_file_name = "scimag_chunk_{}_preface.sql".format(
                    header_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    op_file.write(line)
            elif header:
                op_file_name = "scimag_chunk_header_{}_{}.sql".format(
                    header_no, output_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    output_size += smart_write(line, op_file)
            elif body:
                print("ERROR: Other command in locked state")
            elif footer:
                header_no += 1
                op_file_name = "scimag_chunk_{}_preface.sql".format(
                    header_no)
                with open(
                    "/mnt/n/scimag_2020-05-30.sql/{}".format(op_file_name),
                    "a",
                    encoding="utf-8",
                ) as op_file:
                    op_file.write(line)

            else:
                print("ERROR: Other command in unknown state")