From 240306da20df11550fea7afc235670819eef7ec7 Mon Sep 17 00:00:00 2001 From: Polish Hammer Date: Fri, 14 Aug 2026 15:14:33 +0200 Subject: [PATCH] 3 --- nas-dedup.copy.py | 73 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/nas-dedup.copy.py b/nas-dedup.copy.py index 1d2cec8..c25f56b 100644 --- a/nas-dedup.copy.py +++ b/nas-dedup.copy.py @@ -394,16 +394,63 @@ def scan(root_dir): return files, errors -def apply_filter(files, min_size, keep_exts): +# ---- rejestr pominięć (thread-safe): kategoria -> lista (ścieżka, szczegół) ---- +_SKIP_LOCK = threading.Lock() +_SKIPPED = defaultdict(list) + +def add_skip(category, path, detail=""): + with _SKIP_LOCK: + _SKIPPED[category].append((path, detail)) + +def print_skipped(skipped_list_path=None): + """Wypisuje raport pominięć pogrupowany po kategorii. + Bez skipped_list_path: pełna lista na stdout. + Z skipped_list_path: pełna lista do pliku, na stdout tylko liczniki per kategoria.""" + with _SKIP_LOCK: + if not _SKIPPED: + return + total = sum(len(v) for v in _SKIPPED.values()) + print(f"\n============ POMINIĘTE PLIKI ({total:,}) ============") + if skipped_list_path: + try: + with open(skipped_list_path, "w", encoding="utf-8") as fh: + for cat in sorted(_SKIPPED): + items = _SKIPPED[cat] + fh.write(f"# {cat} ({len(items)})\n") + for path, detail in items: + fh.write(f"{path}\t{detail}\n" if detail else f"{path}\n") + fh.write("\n") + for cat in sorted(_SKIPPED): + print(f" {cat:<28}: {len(_SKIPPED[cat]):,}") + print(f" (pełna lista zapisana do: {skipped_list_path})") + except OSError as e: + print(f" BŁĄD zapisu {skipped_list_path}: {e} — wypisuję na stdout:") + skipped_list_path = None + if not skipped_list_path: + for cat in sorted(_SKIPPED): + items = _SKIPPED[cat] + print(f"\n {cat} ({len(items)}):") + for path, detail in items: + print(f" {path}" + (f" [{detail}]" if detail else "")) + print("========================================") + + +def apply_filter(files, min_size, keep_exts, only_exts=None): kept, skipped, skipped_bytes = [], 0, 0 + skipped_ext = 0 for p, rel, size in files: ext = os.path.splitext(p)[1].lower() + if only_exts is not None and ext not in only_exts: + skipped_ext += 1 + add_skip("poza listą --only-exts", p, f"rozszerzenie '{ext or '(brak)'}'") + continue if size < min_size and ext not in keep_exts: skipped += 1 skipped_bytes += size + add_skip("filtr rozmiaru", p, f"{human(size)} < próg {human(min_size)}") continue kept.append((p, rel, size)) - return kept, skipped, skipped_bytes + return kept, skipped, skipped_bytes, skipped_ext def build_existing_index(existing_files, source_sizes, workers): @@ -458,9 +505,11 @@ def dedup(kept, existing_index, existing_sizes, hash_workers): if key is not None and key in existing_index: dup_existing += 1 + add_skip("już na NAS (--against)", p, f"treść identyczna, {human(size)}") continue if key is not None and key in seen: dup_internal += 1 + add_skip("duplikat w źródle", p, f"kopia wcześniejszego pliku, {human(size)}") continue if key is not None: seen.add(key) @@ -494,6 +543,7 @@ def do_copy(unique, dest, skip_existing, workers): with state_lock: stats["skipped"] += 1 stats["done"] += 1 + add_skip("już skopiowany (rejestr stanu)", p, f"rel={rel}") PROGRESS.tick(add_bytes=size) return dpath = os.path.join(dest, rel) @@ -506,6 +556,7 @@ def do_copy(unique, dest, skip_existing, workers): with state_lock: stats["skipped"] += 1 stats["done"] += 1 + add_skip("już w celu (ten sam rozmiar)", p, f"cel={dpath}") PROGRESS.tick(add_bytes=size) return except OSError: @@ -562,8 +613,12 @@ def main(): help="Katalog z ISTNIEJĄCYMI danymi (np. cały mount NAS). Treść już tam obecna nie jest kopiowana. Można podać wiele razy.") ap.add_argument("--min-size", type=float, default=1.0, help="Próg w MiB; pliki mniejsze pomijane (domyślnie 1). Wyjątki: --keep-exts.") + ap.add_argument("--only-exts", default=None, metavar="EXT,EXT", + help="Kopiuj WYŁĄCZNIE pliki z jednym z podanych rozszerzeń (po przecinku, np. 'mkv,mp4,iso'). Reszta pomijana PRZED hashowaniem. Filtr --min-size dalej obowiązuje.") ap.add_argument("--keep-exts", default="jpg,jpeg,png,gif", help="Rozszerzenia kopiowane niezależnie od rozmiaru (po przecinku).") + ap.add_argument("--skipped-list", default=None, metavar="PLIK", + help="Zapisz pełną listę pominiętych plików do pliku; na stdout tylko liczniki per kategoria. Bez tej opcji pełna lista idzie na stdout.") ap.add_argument("--reserve", type=float, default=1.0, help="Margines wolnego miejsca w GiB do zostawienia na NAS (domyślnie 1).") ap.add_argument("--workers", type=int, default=4, @@ -586,6 +641,11 @@ def main(): dest = os.path.abspath(args.dest) min_size = int(args.min_size * MIB) keep_exts = {("." + e.strip().lower().lstrip(".")) for e in args.keep_exts.split(",") if e.strip()} + only_exts = None + if args.only_exts: + only_exts = {("." + e.strip().lower().lstrip(".")) for e in args.only_exts.split(",") if e.strip()} + if not only_exts: + sys.exit("BŁĄD: --only-exts podane, ale lista rozszerzeń pusta.") reserve = int(args.reserve * (1 << 30)) skip_existing = not args.no_skip_existing workers = max(1, args.workers) @@ -602,6 +662,8 @@ def main(): print(f"Cel : {dest}") if args.against: print(f"Dedup vs : {', '.join(os.path.abspath(a) for a in args.against)}") + if only_exts: + print(f"Tylko ext : {sorted(only_exts)}") print(f"Filtr : pomijaj < {human(min_size)} poza {sorted(keep_exts)}") print(f"Hash : {HASH_NAME}") print(f"Wątki : kopiowanie/against={workers}, hash źródła={hash_workers}") @@ -626,7 +688,8 @@ def main(): total_scanned_bytes = sum(f[2] for f in all_files) print("[2/4] Filtruję...", file=sys.stderr) - kept, skipped_small, skipped_bytes = apply_filter(all_files, min_size, keep_exts) + kept, skipped_small, skipped_bytes, skipped_ext = apply_filter( + all_files, min_size, keep_exts, only_exts) source_sizes = set(f[2] for f in kept) existing_index = set() @@ -658,6 +721,8 @@ def main(): # --- RAPORT --- print("\n================ RAPORT ================") print(f"Plików w źródle : {total_scanned:,} ({human(total_scanned_bytes)})") + if only_exts: + print(f"Pominięte (--only-exts) : {skipped_ext:,}") print(f"Pominięte filtrem (<{human(min_size)}) : {skipped_small:,} ({human(skipped_bytes)})") print(f"Po filtrze : {len(kept):,}") if args.against: @@ -684,6 +749,7 @@ def main(): if args.dry_run: STATE.save(force=True) print("Dry-run: nic nie skopiowano (hashe trafiły do pliku stanu).") + print_skipped(args.skipped_list) return if not args.yes: @@ -703,6 +769,7 @@ def main(): if STATE.path: print(f" w tym z rejestru stanu: {STATE.copy_hits:,}") print("========================================") + print_skipped(args.skipped_list) finally: stop_hb.set() STATE.save(force=True)