Tag: 1.2

Intermediate commits (oldest → newest):
- Music search move file server side att 1
- Deploy!
- Fix rotation and fix of playlist
- Serverside work of search

utils/cut_large_file.py

@@ -15,10 +15,16 @@ the `smart_write` function returns the size of the encoded bytes of the input `l
 return any value.
 """
 
-FILE_SIZE_LIMIT = 100000 * 1024
-SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
-OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
-OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
+FILE_SIZE_LIMIT = 50000 * 1024
+
+
+SOURCE = r"C:\Database\dois-2022-02-12\sci-hub-doi-2022-02-12.txt"
+OUTPUT_FOLDER = r"C:\Database\\chunks\\"
+OUTPUT_FILE_SCHEMA = r"_chunk.txt"
+
+#SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
+#OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
+#OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
 
 
 def smart_write(lines, file):
@@ -33,9 +39,12 @@ def smart_write(lines, file):
     :return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
     bytes.
     """
-    encoded_bytes = lines.encode("utf-8")
-    size_in_bytes = len(encoded_bytes)
-    file.write(lines)
+    try:
+        encoded_bytes = lines.encode("utf-8")
+        size_in_bytes = len(encoded_bytes)
+        file.write(lines)
+    except:
+        return 0
 
     return size_in_bytes
 
@@ -59,8 +68,9 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
     with open(source, encoding="utf-8") as infile:
         output_size = 0
         output_no = 0
-
+        lineno = 0
         for line in infile:
+            lineno += 1
             if output_size > FILE_SIZE_LIMIT:
                 output_no += 1
                 output_size = 0
@@ -70,4 +80,12 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
                 "a",
                 encoding="utf-8",
             ) as op_file:
-                output_size += smart_write(line, op_file)
+                tmp =  smart_write(line, op_file)
+                if tmp > 0:
+                    output_size += smart_write(line, op_file)
+                else:
+                    print (f'Error at {lineno}')
+                    print(line)
+
+if __name__ == "__main__":
+    cut_my_life_into_pieces(SOURCE,OUTPUT_FOLDER,OUTPUT_FILE_SCHEMA)