diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index e0c326c..290d58c 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -65,9 +65,11 @@ class Librarian(object): def check_if_exists(self,page_url): - #TODO: Spowolnic - #TODO: Sprawdzac po mirrorach - #TODO: sprawdzac w tym jebanym pliku + + + + return None + in_scihub_db = True if not page_url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") diff --git a/utils/cut_large_file.py b/utils/cut_large_file.py index bb41bda..6a02153 100644 --- a/utils/cut_large_file.py +++ b/utils/cut_large_file.py @@ -15,10 +15,16 @@ the `smart_write` function returns the size of the encoded bytes of the input `l return any value. """ -FILE_SIZE_LIMIT = 100000 * 1024 -SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql" -OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/" -OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql" +FILE_SIZE_LIMIT = 50000 * 1024 + + +SOURCE = r"C:\Database\dois-2022-02-12\sci-hub-doi-2022-02-12.txt" +OUTPUT_FOLDER = r"C:\Database\\chunks\\" +OUTPUT_FILE_SCHEMA = r"_chunk.txt" + +#SOURCE = "/mnt/n/scimag_2020-05-30.sql/scimag_2020-05-30.sql" +#OUTPUT_FOLDER = "/mnt/n/scimag_2020-05-30.sql/" +#OUTPUT_FILE_SCHEMA = "_scimag_chunk.sql" def smart_write(lines, file): @@ -71,3 +77,6 @@ def cut_my_life_into_pieces(source, output_folder, output_file): encoding="utf-8", ) as op_file: output_size += smart_write(line, op_file) + +if __name__ == "__main__": + cut_my_life_into_pieces(SOURCE,OUTPUT_FOLDER,OUTPUT_FILE_SCHEMA) \ No newline at end of file