Local

conjurer_librarian/conjurer_librarian.py

@@ -65,9 +65,11 @@ class Librarian(object):
 
 
     def check_if_exists(self,page_url):
-        #TODO: Spowolnic
+
-        #TODO: Sprawdzac po mirrorach
+        
-        #TODO: sprawdzac w tym jebanym pliku
+
+        return None
+
         in_scihub_db = True
         if not page_url.startswith(("http:", "https:")):
             raise ValueError("URL must start with 'http:' or 'https:'")

utils/cut_large_file.py

@@ -15,10 +15,16 @@ the `smart_write` function returns the size of the encoded bytes of the input `l
 return any value.
 """
 
-FILE_SIZE_LIMIT = 100000 * 1024
+FILE_SIZE_LIMIT = 50000 * 1024
-SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
+
-OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
+
-OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
+SOURCE = r"C:\Database\dois-2022-02-12\sci-hub-doi-2022-02-12.txt"
+OUTPUT_FOLDER = r"C:\Database\\chunks\\"
+OUTPUT_FILE_SCHEMA = r"_chunk.txt"
+
+#SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql"
+#OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/"
+#OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql"
 
 
 def smart_write(lines, file):
@@ -71,3 +77,6 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
                 encoding="utf-8",
             ) as op_file:
                 output_size += smart_write(line, op_file)
+
+if __name__ == "__main__":
+    cut_my_life_into_pieces(SOURCE,OUTPUT_FOLDER,OUTPUT_FILE_SCHEMA)