From ed51517eb64fd82071f105429f6b741e409ed90e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 20 Nov 2026 03:29:06 +0800 Subject: [PATCH 1/4] Follow changes reads only the folders that changed A change now lists only the folder whose own entries changed and keeps its subfolders (ScanOptions.keep, core/refresh.py), so a file saved directly in the scanned folder no longer rescans the whole scan. All changed folders are read in one worker and refilled in place, so the view keeps its place, expansion and selection. The native feed no longer stops while folders are read. The scope is captured again only when subfolders changed, and the new feed takes over once armed; an NTFS journal feed reads on from where the refresh began. Directory notifications report the real folder, too many changes read their parent folder whole instead of the root, and the five-minute full reconciliation is gone. --- architecture.md | 30 ++- docs/core-api.md | 23 +- docs/guide.md | 2 +- docs/guide_ja.md | 2 +- docs/guide_ko.md | 2 +- docs/guide_zh-CN.md | 2 +- docs/guide_zh-TW.md | 2 +- docs/updates/2026-11.md | 39 +++- docs/updates/README.md | 1 + je_file_tree/core/change_watch.py | 53 ++++- je_file_tree/core/refresh.py | 113 ++++++++++ je_file_tree/core/scanner.py | 23 +- je_file_tree/core/windows_watch.py | 48 ++-- je_file_tree/gui/change_watch.py | 318 +++++++++++++++++++-------- je_file_tree/gui/folder_refresh.py | 139 ++++++++++++ je_file_tree/gui/main_window.py | 19 +- je_file_tree/gui/results_view.py | 35 +++ je_file_tree/gui/strings.py | 24 +- je_file_tree/gui/tree_model.py | 24 +- je_file_tree/gui/worker_lifecycle.py | 15 +- progress.md | 1 + test/test_change_watch.py | 43 +++- test/test_change_watch_gui.py | 124 ++++++++--- test/test_folder_refresh.py | 85 +++++++ test/test_mft_scan.py | 2 +- test/test_shell_dialog_lifecycle.py | 4 +- test/test_worker_lifecycle.py | 32 ++- test/test_workspace_gui.py | 2 +- 28 files changed, 988 insertions(+), 219 deletions(-) create mode 100644 je_file_tree/core/refresh.py create mode 100644 je_file_tree/gui/folder_refresh.py create mode 100644 test/test_folder_refresh.py diff --git a/architecture.md b/architecture.md index 1b173f96..a5ebdc69 100644 --- a/architecture.md +++ b/architecture.md @@ -328,17 +328,27 @@ Native cancellation joins outstanding Windows I/O before freeing storage. Notifi observational rescan requests and never operation authority. Reviewed native CI confirms ordinary and outside-alias events on both Windows USN and Linux inotify, with source preservation/joined cancellation. -`gui.change_watch.FollowChanges` owns one default-off monitor per result tab. Its worker builds maps -and coalesces native events under a lock to avoid unbounded Qt signal queues. The one-second GUI -timer defers dirty folders during scans/analysis, source reviews, modal questions and Undo offers; -bounded ancestor selection reuses rescan_folder and branch replacement. Events continue during -branch scanning, then monitoring joins before tree replacement. Lost scopes/overflow, operation gaps -and five-minute periodic reconciliation force full roots. `rescan_folder` stops the reader as a plain -re-arm (`FollowChanges.stop`), never as an operation gap: that made every followed folder refresh end -in a full-root scan. A finished Trash move, permanent delete or Undo withdraws its own gap request +`gui.change_watch.FollowChanges` owns one default-off monitor per result tab (U-20261120-01). A +`CaptureWorker` (a child thread, so scans and operations wait for it) reads the tree once for the +`WatchScope`; a `ChangeWorker` runs the native feed over it and coalesces events under a lock, so Qt +signal queues stay bounded. The feed reads nothing of the tree and has no Qt parent, so a folder is +read again while it keeps reporting (`queue_worker(readers=...)` lets a scan wait only for the +readers it stopped itself). A report names the folder whose own entries changed (`ChangeBatch.folders`), +or, past `MAX_CHANGED`, the folder holding many changes, read whole (`ChangeBatch.trees`). The +one-second timer defers during scans/analysis, source reviews, modal questions and Undo offers, maps +each path to the deepest scanned folder (`core.node.folder_holding`) and hands them all to +`gui.folder_refresh`: one `ListingWorker` lists each folder with `ScanOptions.keep` (its read +subfolders, by name and identity, stay unread) and `core.refresh.refill` puts the listing into the +same `Node` with the kept subfolders hung back, so a change in the scan root reads only the root's +own entries and the view keeps its place. A kept subfolder replaced meanwhile is read whole next; a +folder removed meanwhile makes its parent's listing be read. When a listing found, lost or replaced a +subfolder, or listed a file with another name, the scope is captured again (otherwise the running feed +still covers it all and carries on) and the new feed takes over only once armed; an NTFS journal feed starts from the position held when +the refresh began (or when a source operation stopped the feed), so nothing between is missed. Only +lost or overflowing feeds and an operation gap without a journal position read the whole root; there is +no periodic full reconciliation. A finished Trash move, permanent delete or Undo withdraws its own gap request (`FollowChanges.resume`), in its tab and, through `MainWindow._tell_peers`, in every other tab of the -OperationGroup, which offers only its own folders holding the changed paths (`core.node.folder_holding`). Branch -rearming preserves the periodic deadline and unrelated pending paths; new roots discard old data. +OperationGroup, which offers only its own folders holding the changed paths. New roots discard old data. Shared OperationGroup exclusion quiesces peer monitors before source mutation/tree changes. Native failures remain visible without retry loops; opt-out/tab close cancel/join through wait_for. diff --git a/docs/core-api.md b/docs/core-api.md index 34ebf600..44afa63a 100644 --- a/docs/core-api.md +++ b/docs/core-api.md @@ -714,16 +714,19 @@ existing USN and Linux inotify, with source preservation and joined cancellation recursive directory notifications. No journals are created. GUI Options → Follow changes is default off and persisted with `follow_changes`. Each tab owns a -ChangeWorker with worker-built captured folder/Node maps and one lock-protected bounded batch; -events never accumulate as Qt signals. The controller polls/coalesces for one second, defers scans -during source reviews/operations, scan/analysis, modal questions and live Undo offers, and invokes -the existing rescan_folder/branch replacement only for captured nodes. Unrelated dirty folders survive -the first refresh. Unknown maps, lost events or hard-link accounting require a full-root scan. -Source-operation gaps and a five-minute periodic reconciliation also require full scans; branch -rearming does not postpone that deadline. Ordinary notifications continue during branch scanning, -then cancel/join before tree replacement/rearming. New roots discard old events; opt-out/tab close -cancel and join through wait_for. Unsupported/incomplete scopes and native errors stay visible in -plain-text status, with no automatic error retry loop. Notifications never grant mutation approval. +CaptureWorker (reads the tree for `captured_scope`) and a ChangeWorker (`watch_scope`, one +lock-protected bounded batch); events never accumulate as Qt signals. The controller polls/coalesces +for one second, defers during source reviews/operations, scan/analysis, modal questions and live +Undo offers, then reads every reported folder's own entries again in one worker +(`scan(path, options=ScanOptions(keep=core.refresh.kept_subfolders(node)))`, then +`core.refresh.refill(node, result.root)`). `ChangeBatch.trees` (too many changes below one folder) +are read whole. The feed keeps reporting during that read; when the read found, lost or replaced a +subfolder (or listed a hard-linked file), the next feed, armed on a fresh scope, replaces it only once +ready, a journal feed reading on `since=` the held `Position`. Lost events and +source-operation gaps without a journal position require a full-root scan; nothing reads the root +on a timer. New roots discard old events; opt-out/tab close cancel and join through wait_for. +Unsupported/incomplete scopes and native errors stay visible in plain-text status, with no automatic +error retry loop. Notifications never grant mutation approval. `background.MonitorConfig(enabled=False, threshold=10, interval_hours=24, roots=())` validates up to 32 explicitly chosen absolute roots, a 1–50% threshold and 1–720-hour schedule. `dumps` / diff --git a/docs/guide.md b/docs/guide.md index f52cee91..5e08777d 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -111,7 +111,7 @@ it, and shows you the biggest folders and files first — in a folder tree, a co of the largest files. When you find something you no longer need, move it to the Recycle Bin right from the window. -In **Options → Follow changes** (off by default), each result tab follows its complete physical Windows/Linux scan independently. A background native USN/directory-notification/inotify feed coalesces bounded changes for a second and rescans only captured affected folders. Unrelated pending branches remain queued; scans, analysis, source-operation reviews, modal questions and live Undo offers defer refresh. Lost events, unknown scopes and source-operation gaps require full scans. A Recycle Bin move, a permanent delete and an Undo, in this tab or another, are the exception: their entries leave the results directly, and only the folders left on offer are rescanned. A refreshed folder is not followed by a full scan; with hard links counted, the whole tree is recounted in memory instead. A five-minute full reconciliation covers observational setup/rearm gaps. Incomplete or combined roots and unsupported scopes cannot start following; native failures remain visible without automatic retry. Partial Windows roots with shared files require readable USN to cover outside aliases. Opt-out and tab close cancel and join the monitor. Notifications grant no source-operation authority. The independent Python API is `change_watch.watch`; native CI verified Windows USN and Linux inotify, including Windows/Linux outside-alias writes. +In **Options → Follow changes** (off by default), each result tab follows its complete physical Windows/Linux scan independently. A background native USN/directory-notification/inotify feed coalesces bounded changes for a second; then FileTree reads again only the folders whose own entries changed, keeping the subfolders they already hold, so a file saved directly in the scanned folder does not rescan the whole scan. Several changed folders are read in one go; a folder with very many changes below it is read whole. The view keeps its place, expanded folders and selection. Scans, analysis, source-operation reviews, modal questions and live Undo offers defer refresh. The feed keeps watching while folders are read and hands over to a fresh one afterwards; an NTFS journal feed reads on from where it was, so nothing in between is missed. Only lost or overflowing events, and a source-operation gap without a journal to read on from, read the whole root again; nothing does so on a timer. A Recycle Bin move, a permanent delete and an Undo, in this tab or another, leave the results directly, and only the folders left on offer are read again. With hard links counted, the whole tree is recounted in memory after a refresh. Incomplete or combined roots and unsupported scopes cannot start following; native failures remain visible without automatic retry. Partial Windows roots with shared files require readable USN to cover outside aliases. Opt-out and tab close cancel and join the monitor. Notifications grant no source-operation authority. The independent Python API is `change_watch.watch` (or `captured_scope` and `watch_scope`); native CI verified Windows USN and Linux inotify, including Windows/Linux outside-alias writes. **Options → Background monitor…** is off by default. Enable it to keep FileTree in the system tray when closing the window; **Quit** always joins all work and exits. If a tray is unavailable the window stays visible; losing a tray restores a hidden window. Once a minute, a worker reads OS available capacity and reports below-threshold crossings (10% by default) through supported system notifications and status/tooltips. OS preferences may suppress notifications; unknown capacity stays unknown. Choose up to 32 folders and a 1–720-hour interval for one-thread gentle scans saved to enabled local history. Scans leave result tabs and sources untouched; incomplete coverage stays explicit. Durable claims across processes prevent duplicate attempts; failed/canceled periods stay throttled and missed periods coalesce once. Foreground scans/source reviews cancel and join gentle work. Disabling joins workers and restores the window. `py -3 start_file_tree.py --background` skips elevation and hides only with already-enabled monitoring and an available tray; the flag never enables it. No automatic clean-up is performed. Core APIs `background.MonitorConfig`, `claim_scan`, `due_roots` and `SpaceWarnings` remain independent. diff --git a/docs/guide_ja.md b/docs/guide_ja.md index 28355487..9bf985db 100644 --- a/docs/guide_ja.md +++ b/docs/guide_ja.md @@ -76,7 +76,7 @@ FileTree がスキャンするのは、ユーザーが求めたときだけで **ディスク容量がどこにあるかを確認します。** FileTree は、フォルダーまたはドライブ全体をスキャンし、その中のすべてのファイルを合計し、フォルダー ツリー、カラフルなツリーマップ、および最大ファイルのリストで、最大のフォルダーとファイルを最初に表示します。不要になったものを見つけたら、ウィンドウから直接ごみ箱に移動します。 -**オプション → 変更を追跡** (デフォルトではオフ) では、各結果タブは完全な物理 Windows/Linux スキャンを個別に追跡します。バックグラウンドのネイティブ USN/directory-notification/inotify フィードは、制限された変更を 1 秒間結合し、キャプチャされた影響を受けるフォルダーのみを再スキャンします。無関係な保留中のブランチはキューに残ったままになります。スキャン、分析、元の項目に対する操作の確認、モーダル質問、およびライブ Undo オファーは更新を延期します。失われたイベント、不明なスコープ、およびソース操作のギャップにはフル スキャンが必要です。 ごみ箱への移動、完全な削除、元に戻す操作は (このタブでも他のタブでも) 例外です。項目は結果から直接取り除かれ、再スキャンされるのは通知に残ったフォルダーだけです。フォルダーを再スキャンした後にフル スキャンは続きません。ハードリンクを数える場合は、代わりにツリー全体をメモリ上で数え直します。 5 分間の完全な調整で、観測のセットアップ/再準備のギャップがカバーされます。不完全または結合されたルートとサポートされていないスコープでは、フォローを開始できません。ネイティブ障害は自動再試行なしで表示されたままになります。共有ファイルを含む部分的な Windows ルートには、外部のエイリアスをカバーするための読み取り可能な USN が必要です。オプトアウトしてタブを閉じてキャンセルし、監視処理の終了を待機します。通知にはソース操作権限は付与されません。独立した Python API は `change_watch.watch` です。ネイティブ CI は、Windows USN および Linux inotify を検証しました (Windows/Linux 外部エイリアス書き込みを含む)。 +**オプション → 変更を追跡** (デフォルトではオフ) では、各結果タブは完全な物理 Windows/Linux スキャンを個別に追跡します。バックグラウンドのネイティブ USN/directory-notification/inotify フィードは制限された変更を 1 秒間まとめ、その後、自身の項目が変わったフォルダーだけを読み直し、すでに持っているサブフォルダーはそのまま残します。そのため、スキャンしたフォルダーに直接保存されたファイルでスキャン全体が読み直されることはありません。変更された複数のフォルダーは一度に読み直し、下で非常に多くの変更があったフォルダーは丸ごと読み直します。表示位置、展開したフォルダー、選択はそのまま残ります。スキャン、分析、元の項目に対する操作の確認、モーダル質問、およびライブ Undo オファーは更新を延期します。フォルダーを読み直す間も監視は続き、その後新しい監視に引き継ぎます。NTFS の変更ジャーナルは元の位置から読み続けるので、その間の変更も漏れません。ルート全体を読み直すのは、イベントの消失やあふれ、および続きを読めるジャーナルがないソース操作のギャップだけで、定期的に全体を読み直すことはありません。ごみ箱への移動、完全な削除、元に戻す操作は (このタブでも他のタブでも) 項目を結果から直接取り除き、読み直すのは通知に残ったフォルダーだけです。ハードリンクを数える場合は、更新後にツリー全体をメモリ上で数え直します。不完全または結合されたルートとサポートされていないスコープでは、フォローを開始できません。ネイティブ障害は自動再試行なしで表示されたままになります。共有ファイルを含む部分的な Windows ルートには、外部のエイリアスをカバーするための読み取り可能な USN が必要です。オプトアウトしてタブを閉じてキャンセルし、監視処理の終了を待機します。通知にはソース操作権限は付与されません。独立した Python API は `change_watch.watch` (または `captured_scope` と `watch_scope`) です。ネイティブ CI は、Windows USN および Linux inotify を検証しました (Windows/Linux 外部エイリアス書き込みを含む)。 **オプション → バックグラウンド モニター…** はデフォルトではオフになっています。有効にすると、ウィンドウを閉じるときに FileTree がシステム トレイに保持されます。 **Quit** は常にすべての処理の終了を待ってから終了します。トレイが使用できない場合、ウィンドウは表示されたままになります。トレイを紛失すると、隠されたウィンドウが復元されます。ワーカーは 1 分に 1 回、OS の利用可能な容量を読み取り、サポートされているシステム通知とステータス/ツールチップを通じて、しきい値を下回ったこと (デフォルトでは 10%) を報告します。 OS の設定により通知が抑制される場合があります。未知の容量は未知のままです。有効なローカル履歴に保存される 1 スレッドのジェントル スキャンの場合、最大 32 個のフォルダーと 1 ~ 720 時間の間隔を選択します。スキャンでは、結果のタブとソースはそのまま残ります。不完全な検査範囲は明示的なままです。プロセス全体にわたる永続的なクレームにより、重複した試行が防止されます。失敗/キャンセルされた期間は調整されたままになり、失敗した期間は 1 回結合されます。通常のスキャンや元の項目に対する操作の確認を開始すると、低負荷のバックグラウンド処理をキャンセルし、その終了を待ちます。無効にすると、バックグラウンド処理の終了を待ち、ウィンドウが復元されます。 `py -3 start_file_tree.py --background` は権限昇格を省略し、すでに有効になっている監視と利用可能なトレイのみを非表示にします。フラグがそれを有効にすることはありません。自動クリーンアップは実行されません。コア API `background.MonitorConfig`、`claim_scan`、`due_roots`、および `SpaceWarnings` は独立したままです。 diff --git a/docs/guide_ko.md b/docs/guide_ko.md index eda55c88..b52a3a4d 100644 --- a/docs/guide_ko.md +++ b/docs/guide_ko.md @@ -76,7 +76,7 @@ FileTree는 사용자가 요청할 때만 스캔합니다. 폴더를 열거나 **디스크 공간이 어디에 있는지 확인하세요.** FileTree는 폴더 또는 전체 드라이브를 검색하고 그 안에 있는 모든 파일을 합산한 다음 가장 큰 폴더와 파일부터 폴더 트리, 다채로운 트리맵 및 가장 큰 파일 목록으로 표시합니다. 더 이상 필요하지 않은 항목을 찾으면 창에서 바로 휴지통으로 이동하세요. -**옵션 → 변경 사항 따르기**(기본적으로 꺼져 있음)에서 각 결과 탭은 전체 물리적 Windows/Linux 스캔을 독립적으로 따릅니다. 백그라운드 네이티브 USN/directory-notification/inotify 피드는 제한된 변경 사항을 잠시 통합하고 캡처된 영향을 받은 폴더만 다시 검색합니다. 관련되지 않은 보류 중인 분기는 대기열에 남아 있습니다. 스캔, 분석, 소스 작업 검토, 모달 질문 및 실시간 실행 취소는 새로 고침을 연기합니다. 손실된 이벤트, 알 수 없는 범위 및 소스 작업 격차에는 전체 검사가 필요합니다. 휴지통으로 이동, 영구 삭제, 실행 취소는 (이 탭이든 다른 탭이든) 예외입니다. 항목은 결과에서 바로 제거되고, 알림에 남은 폴더만 다시 검색합니다. 폴더를 다시 검색한 뒤 전체 검사가 이어지지 않으며, 하드 링크를 계산할 때는 대신 메모리에서 트리 전체를 다시 계산합니다. 5분 동안의 전체 조정은 관찰 설정/재준비 공백을 다룹니다. 불완전하거나 결합된 루트 및 지원되지 않는 범위는 다음을 시작할 수 없습니다. 운영체제 오류는 자동 재시도 없이도 계속 표시됩니다. 공유 파일이 있는 부분 Windows 루트에는 외부 별칭을 포함하기 위해 읽기 가능한 USN이 필요합니다. 옵트아웃 및 탭 닫기를 취소하고 모니터 작업 종료를 기다림하세요. 알림은 소스 작업 권한을 부여하지 않습니다. 독립적인 Python API는 `change_watch.watch`입니다. 운영체제 CI는 Windows/Linux 외부 별칭 쓰기를 포함하여 Windows USN 및 Linux inotify를 확인했습니다. +**옵션 → 변경 사항 따르기**(기본적으로 꺼져 있음)에서 각 결과 탭은 전체 물리적 Windows/Linux 스캔을 독립적으로 따릅니다. 백그라운드 네이티브 USN/directory-notification/inotify 피드는 제한된 변경 사항을 1초 동안 모은 뒤, 자체 항목이 바뀐 폴더만 다시 읽고 이미 가진 하위 폴더는 그대로 둡니다. 그래서 스캔한 폴더에 바로 저장된 파일 때문에 스캔 전체를 다시 읽지 않습니다. 변경된 여러 폴더는 한 번에 읽고, 아래에 변경이 매우 많은 폴더는 통째로 다시 읽습니다. 보기의 위치, 펼친 폴더와 선택은 그대로 유지됩니다. 스캔, 분석, 소스 작업 검토, 모달 질문 및 실시간 실행 취소는 새로 고침을 연기합니다. 폴더를 다시 읽는 동안에도 감시는 계속되며, 그 뒤 새 감시에 넘겨줍니다. NTFS 변경 저널은 원래 위치부터 이어서 읽으므로 그 사이의 변경도 놓치지 않습니다. 이벤트가 손실되거나 넘친 경우와 이어 읽을 저널이 없는 소스 작업 공백만 루트 전체를 다시 읽으며, 주기적으로 전체를 다시 읽지는 않습니다. 휴지통으로 이동, 영구 삭제, 실행 취소는 (이 탭이든 다른 탭이든) 항목을 결과에서 바로 제거하고, 알림에 남은 폴더만 다시 읽습니다. 하드 링크를 계산할 때는 업데이트 후 메모리에서 트리 전체를 다시 계산합니다. 불완전하거나 결합된 루트 및 지원되지 않는 범위는 다음을 시작할 수 없습니다. 운영체제 오류는 자동 재시도 없이도 계속 표시됩니다. 공유 파일이 있는 부분 Windows 루트에는 외부 별칭을 포함하기 위해 읽기 가능한 USN이 필요합니다. 옵트아웃 및 탭 닫기를 취소하고 모니터 작업 종료를 기다림하세요. 알림은 소스 작업 권한을 부여하지 않습니다. 독립적인 Python API는 `change_watch.watch`(또는 `captured_scope`와 `watch_scope`)입니다. 운영체제 CI는 Windows/Linux 외부 별칭 쓰기를 포함하여 Windows USN 및 Linux inotify를 확인했습니다. **옵션 → 백그라운드 모니터…**은 기본적으로 꺼져 있습니다. 창을 닫을 때 시스템 트레이에 FileTree를 유지하려면 이를 활성화하십시오. **Quit**은 항상 모든 작업이 끝날 때까지 기다린 후 종료합니다. 트레이를 사용할 수 없는 경우 창은 계속 표시됩니다. 트레이를 잃어버리면 숨겨진 창이 복원됩니다. 1분에 한 번씩 작업자는 OS의 사용 가능한 용량을 읽고 지원되는 시스템 알림 및 상태/도구 설명을 통해 임계값 미만 초과(기본적으로 10%)를 보고합니다. OS 기본 설정은 알림을 표시하지 않을 수 있습니다. 알 수 없는 용량은 알 수 없는 상태로 유지됩니다. 활성화된 로컬 기록에 저장되는 단일 스레드 젠틀 스캔에 대해 최대 32개의 폴더와 1~720시간 간격을 선택합니다. 스캔 결과 탭과 소스는 그대로 유지됩니다. 불완전한 적용 범위는 명시적으로 유지됩니다. 프로세스 전반에 걸친 영구 실행 기록은 중복 시도를 방지합니다. 실패/취소된 기간은 제한된 상태로 유지되고 누락된 기간은 한 번 통합됩니다. 일반 스캔이나 원본 작업 검토를 시작하면 저부하 백그라운드 작업을 취소하고 종료를 기다립니다. 비활성화하면 백그라운드 작업이 끝날 때까지 기다리고 창이 복원됩니다. `py -3 start_file_tree.py --background`은 권한 상승을 생략하고 이미 활성화된 모니터링과 사용 가능한 트레이에서만 숨깁니다. 플래그는 이를 활성화하지 않습니다. 자동 정리는 수행되지 않습니다. 핵심 API `background.MonitorConfig`, `claim_scan`, `due_roots` 및 `SpaceWarnings`은 독립적으로 유지됩니다. diff --git a/docs/guide_zh-CN.md b/docs/guide_zh-CN.md index 75d464c8..39ec6c2b 100644 --- a/docs/guide_zh-CN.md +++ b/docs/guide_zh-CN.md @@ -76,7 +76,7 @@ FileTree 只在要求时才扫描:打开或拖入文件夹、点击“重新 **看看磁盘空间都用到哪里去了。** FileTree 会扫描一个文件夹或整个磁盘,把里面每个文件的大小加总,先列出最大的文件夹和文件——用文件夹树、彩色的方块图和最大文件列表呈现。找到不需要的东西时,直接在窗口里把它移到回收站。 -**选项 → 跟随变更** 默认关闭,每个结果选项卡可独立跟随完整、实际的 Windows/Linux 扫描。后台原生 USN/目录通知/inotify 将数量受限的变化合并一秒,只重扫已记录的受影响文件夹,其他待更新分支保留在队列。扫描、分析、来源操作审核、询问窗口及有效撤销提示期间延后更新。事件丢失、未知范围及来源操作间隔须完整重扫,每五分钟再完整核对,涵盖设置/重新启用的观察间隔。移到回收站、永久删除与撤销(无论在这个还是其他选项卡)例外:项目直接从结果移除,只重扫提示栏中留下的文件夹。重扫一个文件夹之后不会再完整重扫;计入硬链接时改为在内存中重新计算整棵树。不完整、合并或不支持的范围不能启用;原生错误持续显示,不自动重试。Windows 部分范围含共享文件时须可读取 USN,才能涵盖范围外别名。关闭选项或选项卡会取消并等待监看任务结束。通知不授予来源操作权限。独立 Python API 为 `change_watch.watch`;原生 CI 已验证 Windows USN 与 Linux inotify,包含 Windows/Linux 范围外别名写入。 +**选项 → 跟随变更** 默认关闭,每个结果选项卡可独立跟随完整、实际的 Windows/Linux 扫描。后台原生 USN/目录通知/inotify 将数量受限的变化合并一秒,之后只重新读取自身项目有变化的文件夹,并保留其中已有的子文件夹;因此直接存进扫描文件夹的文件不会让整个扫描重扫。多个变化的文件夹一次读完;下面变化非常多的文件夹则整个重读。视图会保留原来的位置、展开的文件夹与选择。扫描、分析、来源操作审核、询问窗口及有效撤销提示期间延后更新。读取文件夹期间监看持续进行,之后交接给新的监看;NTFS 变更日志会从原来的位置继续读,中间的变化不会漏掉。只有事件丢失或溢出,以及没有变更日志可接续的来源操作间隔,才会重新读取整个根目录;不会定时完整重扫。移到回收站、永久删除与撤销(无论在这个还是其他选项卡)会直接从结果移除项目,只重新读取提示栏中留下的文件夹。计入硬链接时,更新后在内存中重新计算整棵树。不完整、合并或不支持的范围不能启用;原生错误持续显示,不自动重试。Windows 部分范围含共享文件时须可读取 USN,才能涵盖范围外别名。关闭选项或选项卡会取消并等待监看任务结束。通知不授予来源操作权限。独立 Python API 为 `change_watch.watch`(或 `captured_scope` 与 `watch_scope`);原生 CI 已验证 Windows USN 与 Linux inotify,包含 Windows/Linux 范围外别名写入。 **选项 → 后台监视器…** 默认关闭。启用后,关闭窗口会将 FileTree 保留在托盘;**退出** 一定等待全部任务并结束。没有托盘时窗口保持显示,托盘消失也会恢复隐藏窗口。后台任务每分钟读取系统可用容量,低于阈值时(默认 10%)通过支持的系统通知与状态/提示显示;系统偏好设置可能停用通知,未知容量保持未知。可选最多 32 个文件夹及 1–720 小时间隔,以单个低优先级线程扫描并保存至已启用的本地历史。扫描不改动结果选项卡或来源,不完整范围保持明确。跨进程持久记录防止重复尝试;失败/取消的期间不重跑,错过多次计划只合并一次。前台扫描/来源审核会取消并等待低优先级任务。停用也等待任务并恢复窗口。`py -3 start_file_tree.py --background` 不询问提升权限,仅在监视已启用且托盘可用时隐藏;标志不会启用监视。不自动清理。核心 API `background.MonitorConfig`、`claim_scan`、`due_roots` 及 `SpaceWarnings` 保持独立。 diff --git a/docs/guide_zh-TW.md b/docs/guide_zh-TW.md index 21ee078a..0b4c6b77 100644 --- a/docs/guide_zh-TW.md +++ b/docs/guide_zh-TW.md @@ -76,7 +76,7 @@ FileTree 只在要求時才掃描:開啟或拖入資料夾、按「重新掃 **看看磁碟空間都用到哪裡去了。** FileTree 會掃描一個資料夾或整顆磁碟,把裡面每個檔案的大小加總,先列出最大的資料夾和檔案——用資料夾樹、彩色的方塊圖和最大檔案清單呈現。找到不需要的東西時,直接在視窗裡把它移到資源回收筒。 -**選項 → 跟隨變更** 預設關閉,每個結果分頁可獨立跟隨完整、實際的 Windows/Linux 掃描。背景原生 USN/目錄通知/inotify 將數量受限的變動合併一秒,只重掃已記錄的受影響資料夾,其他待更新分支保留在佇列。掃描、分析、來源操作審查、詢問視窗及有效復原提示期間延後更新。事件遺失、未知範圍及來源操作間隔須完整重掃,每五分鐘再完整核對,涵蓋設定/重新啟用的觀察間隔。移到資源回收筒、永久刪除與復原(不論在這個或其他分頁)例外:項目直接從結果移除,只重掃提示列中留下的資料夾。重掃一個資料夾之後不會再完整重掃;計入硬連結時改為在記憶體中重新計算整棵樹。不完整、合併或不支援的範圍不能啟用;原生錯誤持續顯示,不自動重試。Windows 部分範圍含共用檔案時須可讀取 USN,才能涵蓋範圍外別名。關閉選項或分頁會取消並等待監看工作結束。通知不授予來源操作權限。獨立 Python API 為 `change_watch.watch`;原生 CI 已驗證 Windows USN 與 Linux inotify,包含 Windows/Linux 範圍外別名寫入。 +**選項 → 跟隨變更** 預設關閉,每個結果分頁可獨立跟隨完整、實際的 Windows/Linux 掃描。背景原生 USN/目錄通知/inotify 將數量受限的變動合併一秒,之後只重新讀取自身項目有變動的資料夾,並保留其中已有的子資料夾;因此直接存進掃描資料夾的檔案不會讓整個掃描重掃。多個變動的資料夾一次讀完;底下變動非常多的資料夾則整個重讀。檢視會保留原本的位置、展開的資料夾與選取。掃描、分析、來源操作審查、詢問視窗及有效復原提示期間延後更新。讀取資料夾期間監看持續進行,之後交接給新的監看;NTFS 變更日誌會從原本的位置繼續讀,中間的變動不會漏掉。只有事件遺失或溢位,以及沒有變更日誌可接續的來源操作間隔,才會重新讀取整個根目錄;不會定時完整重掃。移到資源回收筒、永久刪除與復原(不論在這個或其他分頁)會直接從結果移除項目,只重新讀取提示列中留下的資料夾。計入硬連結時,更新後在記憶體中重新計算整棵樹。不完整、合併或不支援的範圍不能啟用;原生錯誤持續顯示,不自動重試。Windows 部分範圍含共用檔案時須可讀取 USN,才能涵蓋範圍外別名。關閉選項或分頁會取消並等待監看工作結束。通知不授予來源操作權限。獨立 Python API 為 `change_watch.watch`(或 `captured_scope` 與 `watch_scope`);原生 CI 已驗證 Windows USN 與 Linux inotify,包含 Windows/Linux 範圍外別名寫入。 **選項 → 背景監視器…** 預設關閉。啟用後,關閉視窗會將 FileTree 保留在系統匣;**結束** 一定等待全部工作並退出。沒有系統匣時視窗保持顯示,系統匣消失也會恢復隱藏視窗。背景工作每分鐘讀取系統可用容量,低於門檻時(預設 10%)透過支援的系統通知與狀態/提示顯示;系統偏好設定可能停用通知,未知容量維持未知。可選最多 32 個資料夾及 1–720 小時間隔,以單一低優先順序執行緒掃描並儲存至已啟用的本機歷史。掃描不改動結果分頁或來源,不完整範圍保持明確。跨程序持久記錄防止重複嘗試;失敗/取消的期間不重跑,錯過多次排程只合併一次。前景掃描/來源審查會取消並等待低優先順序工作。停用也等待工作並恢復視窗。`py -3 start_file_tree.py --background` 不詢問提升權限,僅在監視已啟用且系統匣可用時隱藏;旗標不會啟用監視。不自動清理。核心 API `background.MonitorConfig`、`claim_scan`、`due_roots` 及 `SpaceWarnings` 保持獨立。 diff --git a/docs/updates/2026-11.md b/docs/updates/2026-11.md index ae6f52fd..c5e0b8b3 100644 --- a/docs/updates/2026-11.md +++ b/docs/updates/2026-11.md @@ -466,7 +466,7 @@ - 新增 `FollowChanges.resume`:撤回該要求並重新開始監看。停止前已回報的資料夾,以及其他原因(事件遺失、未知範圍、定期核對)的整棵要求都保留。 - `MainWindow._trash_finished`、`_delete_finished` 與 `TrashUndo._finished` 在結束時呼叫 `resume`。成功的項目已從結果移除;略過/失敗/拒絕的批次與復原仍由 `_mark_changed` 只重掃受影響的資料夾。 - 其他來源操作(搬移/重新命名、複製、連結重複檔案、壓縮)不變:它們本來就標記整個根目錄。 -- **取捨**:監看停止期間其他程式造成的變動,改由每五分鐘的完整核對發現,而不是操作結束後立刻重掃。 +- **取捨**:監看停止期間其他程式造成的變動,改由每五分鐘的完整核對發現,而不是操作結束後立刻重掃。→ 定時完整核對已在 U-20261120-01 移除。 - **未變更**:「計入硬連結」模式下刪除仍無法就地更新(計入總量須重新計算),受影響的資料夾留在提示列,重新掃描時會掃整個根目錄。同一工作區其他分頁的監看也會因操作而停止,仍會各自重掃整個根目錄。上述 `_update_texts` 的耗時未處理。 - **驗證**:`test/test_change_watch_gui.py`(新增 `test_a_delete_shown_in_place_does_not_request_the_whole_root` 兩種情況)、`test_worker_lifecycle.py`、`test_permanent_delete_gui.py`、`test_workspace_gui.py`、`test_gui.py`、`test_readme_parity.py`;`ruff check .`。`test_trash_undo_gui.py` 在這台機器上無法執行:第一個測試的移到資源回收筒被略過並停在警告視窗,未修改的程式碼同樣如此,因此復原路徑的 `resume` 呼叫本機未驗證,由 CI 執行。沒有在可見視窗與真實的 USN 監看下操作驗證(測試以替身取代監看來源)。 - **檔案**:`je_file_tree/gui/change_watch.py`、`gui/main_window.py`、`gui/trash_undo.py`;`test/test_change_watch_gui.py`、`test/test_worker_lifecycle.py`;`architecture.md`、`docs/guide.md` 與四種語言的翻譯。 @@ -695,3 +695,40 @@ - **未驗證**:只在隱藏視窗中量測,沒有在使用者的螢幕上量測。 - **檔案**:`je_file_tree/core/scanner.py`、`core/cleanup.py`;`architecture.md`;`test/test_pacing.py`。 - **待辦**:無。 + +## U-20261120-01 · 2026-11-20 · 「跟隨變更」只重新讀取變動的資料夾,不再重掃整個掃描 · #perf #gui #scan + +- **原因**:使用者要求「變動的時候只掃描那資料夾不要整個重掃,可以接合 watchdog」。檢查既有的「跟隨變更」,以下情況會重掃整個根目錄或整個子樹: + - 每五分鐘定時完整核對一次根目錄。 + - 更新一個資料夾就是重掃它的整個子樹;變動發生在掃描的根目錄本身時(例如直接存進掃描資料夾的檔案),等於整個重掃。 + - 一批變動超過 256 個資料夾就改為重掃整個根目錄。 + - 回報的路徑不在監看當時記下的資料夾表中(例如更新後才出現的資料夾),也改為重掃整個根目錄。 + - 每次更新資料夾前都停止監看、更新後再重新啟動,中間的變動只能靠定時核對補上。 +- **變更**: + - `ScanOptions.keep` 與新的 `core/refresh.py`:重新列出資料夾自身的項目,已讀過的子資料夾依名稱與檔案識別碼保留、不再讀取(標記為 `KEPT`);`refill` 把結果放回同一個 `Node`,掛回保留的子資料夾,並修正上層總計。識別碼不同(已被替換)的子資料夾會整個重讀;更新期間被移除的資料夾,改為重新讀取包含它的資料夾。 + - `gui/folder_refresh.py`:一個背景工作一次讀完所有變動的資料夾。資料夾仍是同一個節點,樹狀檢視保留位置、展開的資料夾與選取。 + - 監看拆成讀取範圍的 `CaptureWorker`(讀取樹,掃描與操作會等它)與執行原生監看的 `ChangeWorker`(不讀樹、沒有 Qt 父物件)。讀取資料夾期間監看持續回報;`queue_worker(readers=...)` 讓資料夾掃描只等它自己暫停的背景工作。 + - 只有在讀到的子資料夾有增減或替換、或讀到有其他名稱(硬連結)的檔案時,才重新讀取監看範圍;新的監看就緒後才取代舊的。NTFS USN 監看從更新開始時(或來源操作停止監看時)的位置接著讀,中間的變動不會漏掉;有這個位置時,來源操作的間隔不必再重掃整個根目錄。 + - 目錄通知改為回報實際變動的資料夾路徑,由視窗對應到掃描中最深的資料夾(`folder_holding`);比掃描新的資料夾由其上層的列出結果整個讀取。 + - 一批超過 256 個路徑時,最深的路徑改為其上層資料夾並整個重讀(`ChangeBatch.trees`),而不是整個根目錄。 + - 移除每五分鐘的完整核對。 +- **watchdog**:未採用。已安裝 watchdog 7.1.0 檢查原始碼:Windows 的 ReadDirectoryChangesW 緩衝區溢位時傳回 0 位元組,watchdog 不產生任何事件;macOS FSEvents 也不處理「須重新掃描子資料夾」的旗標。兩者都會無聲遺失變動,而 FileTree 現有的原生監看會回報遺失並重掃。Windows 與 Linux 已有原生 USN/目錄通知/inotify。 +- **量測**(C:\Windows,375,625 個項目,不開視窗,兩次): + + | 情況 | 之前 | 現在 | + |---|---|---| + | 根目錄有一個檔案變動 | 整個重掃 15.1–20.0 s | 列出根目錄 4–5 ms,放回 0.9–1.1 ms | + | System32 有一個檔案變動 | 重掃 System32 整個子樹 0.41–0.47 s | 列出 System32 本身並放回 98–121 ms | + | 定時完整核對 | 每五分鐘整個重掃一次 | 無 | + + 重新讀取監看範圍在背景要 4.7–5.4 s。之前每次更新資料夾後都會重讀;現在只在子資料夾有變化時才重讀。 +- **取捨**: + - 監看因來源操作停止時,如果不是 USN(目錄通知或 inotify),結束後仍須重掃整個根目錄(與之前相同)。 + - inotify:更新時新發現的資料夾,在新監看就緒之前的幾秒內,裡面的變動不會被看到(之前由定時核對補上)。 + - 更新後的短暫期間會同時開著新舊兩個監看。 +- **驗證**: + - 新增 `test/test_folder_refresh.py`(5 個);`test_change_watch.py` 新增 USN 位置交接與批次上限 2 個;`test_change_watch_gui.py` 新增根目錄只讀自身、讀取期間持續回報並交接、讀取前資料夾被移除 3 個,並改寫原本預期整個重掃或子樹重掃的案例。 + - 相關的 61 個測試檔案共 872 個測試通過(排除 1 個已知的長路徑測試,略過 9 個);`ruff check .` 通過。 +- **未驗證**:USN 位置交接只用測試替身驗證(本機需要系統管理員才能讀取變更日誌);Linux inotify 的新舊監看並存(監看數量上限)只由 CI 的原生測試涵蓋;沒有在使用者的電腦上操作。 +- **檔案**:`je_file_tree/core/refresh.py`(新)、`core/scanner.py`、`core/change_watch.py`、`core/windows_watch.py`;`je_file_tree/gui/folder_refresh.py`(新)、`gui/change_watch.py`、`gui/main_window.py`、`gui/results_view.py`、`gui/tree_model.py`、`gui/worker_lifecycle.py`、`gui/strings.py`;`architecture.md`、`docs/core-api.md`、`docs/guide.md` 與四個翻譯;`progress.md`;測試檔案。 +- **待辦**:原生環境驗證記錄於 `progress.md` #144。 diff --git a/docs/updates/README.md b/docs/updates/README.md index bac1d136..c2b8d9c7 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -70,6 +70,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261120-01 | 2026-11-20 | 「跟隨變更」只重新讀取變動的資料夾,不再重掃整個掃描 | #perf #gui #scan | [2026-11](2026-11.md) | | U-20261119-07 | 2026-11-19 | 大型資料夾的加總、排序與清理比對也分頁讓路給視窗 | #perf #fix | [2026-11](2026-11.md) | | U-20261119-06 | 2026-11-19 | 只拖動捲軸也會卡頓:清理建議背景工作沒有讓路給視窗 | #perf #gui #fix | [2026-11](2026-11.md) | | U-20261119-05 | 2026-11-19 | 掃描結束後清理建議出現時,圖表卡住約 0.7 秒 | #perf #gui #fix | [2026-11](2026-11.md) | diff --git a/je_file_tree/core/change_watch.py b/je_file_tree/core/change_watch.py index 9a4913eb..a8e06187 100644 --- a/je_file_tree/core/change_watch.py +++ b/je_file_tree/core/change_watch.py @@ -4,6 +4,7 @@ from collections.abc import Callable from dataclasses import dataclass +import os import stat import sys import threading @@ -21,11 +22,16 @@ @dataclass(frozen=True, slots=True) class ChangeBatch: - """Captured folder paths needing fresh scans, or a full-root request after loss/uncertainty.""" + """Folders whose own entries changed, folders to read whole, or a full-root request after loss. + + ``folders`` and ``trees`` are reported paths inside the scan, never targets: the window maps each to + the deepest folder it scanned there (``node.folder_holding``) and reads that folder again. + """ folders: tuple[str, ...] = () full: bool = False reason: str = "" + trees: tuple[str, ...] = () @dataclass(frozen=True, slots=True) @@ -112,15 +118,47 @@ def check_folder(folder: WatchFolder) -> None: raise ValueError("Watched folder identity changed") -def changed(folders: set[str], *, full: bool = False, reason: str = "") -> ChangeBatch: - """Bound an event batch; too many affected folders conservatively request a full scan.""" - if full or len(folders) > MAX_CHANGED: +def changed(folders: set[str], *, full: bool = False, reason: str = "", trees: set[str] | frozenset[str] = frozenset() + ) -> ChangeBatch: + """Bound an event batch; past ``MAX_CHANGED`` paths, the deepest give way to their parent folders, read whole. + + A build touching a thousand folders refreshes the few folders holding them instead of the whole scan. + """ + if full: return ChangeBatch(full=True, reason=reason or "overflow") - return ChangeBatch(tuple(sorted(folders)), reason=reason) + folders, trees = set(folders), set(trees) + while len(folders) + len(trees) > MAX_CHANGED: + deepest = max(map(_depth, folders | trees)) + if not deepest: + return ChangeBatch(full=True, reason=reason or "overflow") # only drive roots are left + lifted = {os.path.dirname(path) for path in folders | trees if _depth(path) == deepest} + folders = {path for path in folders if _depth(path) < deepest} - lifted + trees = {path for path in trees if _depth(path) < deepest} | lifted + return ChangeBatch(tuple(sorted(folders)), reason=reason, trees=tuple(sorted(trees))) + + +def _depth(path: str) -> int: + return path.rstrip("\\/").count(os.sep) + + +def merged(first: ChangeBatch, second: ChangeBatch) -> ChangeBatch: + """Both batches as one, bounded like ``changed``.""" + return changed(set(first.folders) | set(second.folders), full=first.full or second.full, + reason=second.reason or first.reason, trees=set(first.trees) | set(second.trees)) + + +Position = tuple[int, int] # an NTFS change journal and the next record to read from it def watch(root: Node, on_change: Callable[[ChangeBatch], None], cancel: threading.Event, *, ready: Callable[[str], None] | None = None) -> None: + """``captured_scope`` then ``watch_scope``: a joinable native feed for ``root``.""" + watch_scope(captured_scope(root, cancel), on_change, cancel, ready=ready) + + +def watch_scope(scope: WatchScope, on_change: Callable[[ChangeBatch], None], cancel: threading.Event, # noqa: PLR0913 + *, ready: Callable[[str], None] | None = None, since: Position | None = None, + moved: Callable[[Position], None] | None = None) -> None: """Joinable native metadata feed; caller must rescan and separately revalidate later operations. Windows uses an existing NTFS USN journal when readable, otherwise one recursive directory @@ -128,8 +166,9 @@ def watch(root: Node, on_change: Callable[[ChangeBatch], None], cancel: threadin No journal creation/deletion, payload reads, Qt watchers or filesystem mutations are involved. Overflow/gaps request full rescans; unsupported/failed feeds raise visible OSError/ValueError. Setup/rearm and concurrent scans remain observational rather than a consistent live snapshot. + A journal feed reports its read position to ``moved``; one started ``since`` such a position reads + on from there, so a feed armed after a refresh misses nothing that happened while it was armed. """ - scope = captured_scope(root, cancel) folders = scope.folders if not folders or cancel.is_set(): return @@ -137,7 +176,7 @@ def watch(root: Node, on_change: Callable[[ChangeBatch], None], cancel: threadin if sys.platform == "win32": from je_file_tree.core.windows_watch import watch_windows # noqa: PLC0415 - native backend - watch_windows(folders, on_change, cancel, ready, scope.shared) + watch_windows(folders, on_change, cancel, ready, scope.shared, since=since, moved=moved) elif sys.platform.startswith("linux"): from je_file_tree.core.linux_watch import watch_linux # noqa: PLC0415 - native backend diff --git a/je_file_tree/core/refresh.py b/je_file_tree/core/refresh.py new file mode 100644 index 00000000..c8584a17 --- /dev/null +++ b/je_file_tree/core/refresh.py @@ -0,0 +1,113 @@ +"""Put a fresh listing of one scanned folder in place, keeping the subfolders it did not read again. + +Follow changes learns which folders' own entries changed. Reading such a folder with +``ScanOptions(keep=kept_subfolders(folder))`` lists it again but leaves each subfolder it already holds +unread (marked ``KEPT``); ``refill`` then puts that listing into the same ``Node`` and hangs the kept +subfolders back, so a change directly in the scanned folder no longer reads the whole scan again. +A kept subfolder is matched by name and file identity, so one that was replaced is read instead. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterable + +from je_file_tree.core.node import Node +from je_file_tree.core.scanner import EXCLUDED, HIDDEN_OMITTED, KEPT, PARTIAL_FOLDER +from je_file_tree.core.snapshot import unpack_snapshot + +_KEEPABLE = (None, PARTIAL_FOLDER, HIDDEN_OMITTED) # read folders; one that failed or was stopped is read again + + +def kept_subfolders(folder: Node) -> frozenset[tuple[str, int, int]]: + """``(name, device, file number)`` of the read subfolders directly in ``folder`` that a refresh may keep.""" + keep = set() + for child in folder.children: + if child.is_dir and not child.is_link and child.error in _KEEPABLE and child.snapshot is not None: + device, number = unpack_snapshot(child.snapshot).identity + if number: + keep.add((child.name, device, number)) + return frozenset(keep) + + +def _identity(node: Node) -> tuple[int, int] | None: + return None if node.snapshot is None else unpack_snapshot(node.snapshot).identity + + +def refill(folder: Node, fresh: Node) -> frozenset[str] | None: + """Make ``folder`` hold ``fresh`` (a listing of it), correcting every total above it. + + ``folder`` keeps its identity, so the window's place in the tree and a root stay valid; its other + old entries are detached. Returns the names of the subfolders kept, or None with nothing changed + when a ``KEPT`` entry has no subfolder of the same name and identity left in ``folder``. + """ + if not isinstance(folder.children, list) or not isinstance(fresh.children, list): + return None + known = {child.name: child for child in folder.children if child.is_dir and not child.is_link} + swaps = [] + for index, child in enumerate(fresh.children): + if child.error != KEPT: + continue + old = known.get(child.name) + if old is None or _identity(old) is None or _identity(old) != _identity(child): + return None + swaps.append((index, old)) + children = fresh.children + for index, old in swaps: + children[index] = old + kept = {id(old) for _, old in swaps} + for child in folder.children: + if id(child) not in kept: + child.parent = None + for child in children: + child.parent = folder + before = folder.size, folder.allocated, folder.file_count, folder.dir_count + own = 0.0 if folder.parent is None or fresh.snapshot is None else unpack_snapshot(fresh.snapshot).modified_ns / 1e9 + folder.children = children + folder.snapshot, folder.error = fresh.snapshot, fresh.error + _total(folder, own) + _correct_above(folder, before) + return frozenset(old.name for _, old in swaps) + + +def _total(folder: Node, own: float) -> None: + """Add up ``folder``'s direct entries the way the scanner does and sort them largest first.""" + children = folder.children + folder.size = sum(child.size for child in children) + folder.allocated = sum(child.allocated for child in children) + folder.file_count = sum(child.file_count for child in children) + folder.dir_count = sum(1 + child.dir_count for child in children + if child.is_dir and not child.is_link and child.error != EXCLUDED) + folder.modified = max([own, *(child.modified for child in children)]) + if isinstance(children, list): + children.sort(key=lambda child: (-child.size, child.name.lower())) + + +def _correct_above(folder: Node, before: tuple[int, int, int, int]) -> None: + size, allocated = folder.size - before[0], folder.allocated - before[1] + files, folders = folder.file_count - before[2], folder.dir_count - before[3] + node = folder.parent + while node is not None: + node.size += size + node.allocated += allocated + node.file_count += files + node.dir_count += folders + node.modified = max(node.modified, folder.modified) + node = node.parent + + +def refreshed_errors(errors: Iterable[tuple[str, str]], folder: str, kept: frozenset[str], + fresh: Iterable[tuple[str, str]]) -> list[tuple[str, str]]: + """``errors`` without those a refill of ``folder`` replaced, plus the listing's own ``fresh`` ones. + + What was recorded inside a kept subfolder stays; ``folder`` itself and everything else in it was + read again. + """ + prefix = folder.rstrip("\\/") + os.sep + + def replaced(path: str) -> bool: + if path == folder: + return True + return path.startswith(prefix) and path[len(prefix):].split(os.sep, 1)[0] not in kept + + return [error for error in errors if not replaced(error[0])] + list(fresh) diff --git a/je_file_tree/core/scanner.py b/je_file_tree/core/scanner.py index 8bb61a11..4a93c06f 100644 --- a/je_file_tree/core/scanner.py +++ b/je_file_tree/core/scanner.py @@ -68,6 +68,7 @@ EXCLUDED = "excluded: skipped by the exclusions" HIDDEN_OMITTED = "excluded: hidden entries omitted" PARTIAL_FOLDER = "incomplete: some entries could not be read" +KEPT = "kept: read before and not read again" # a subfolder ``ScanOptions.keep`` names (``core/refresh.py``) class ScanCancelledError(Exception): @@ -106,6 +107,10 @@ class ScanOptions: # Read a whole NTFS drive from its master file table where that is possible (``core/mft_scan.py``); # like ``adaptive`` it is this machine's own choice and takes no part in comparisons. mft: bool = field(default=False, compare=False) + # Direct subfolders of the scanned folder that are listed but not read, as ``(name, device, file + # number)``: Follow changes reads one changed folder's own entries again and keeps what it already + # holds below them (``core/refresh.py``). A listing of one folder, not a choice: never compared. + keep: frozenset[tuple[str, int, int]] = field(default=frozenset(), compare=False) @dataclass(frozen=True, slots=True) @@ -383,9 +388,12 @@ def _take(self) -> tuple[Node, str] | None: return self._pending.pop() def _finish(self, folder: Node, path: str, read: _FolderRead) -> None: + subfolders = read.subfolders + if self._options.keep and folder.parent is None: + subfolders = _unkept(subfolders, self._options.keep) with self._condition: - self._pending.extend(read.subfolders) - self.folders.extend(node for node, _ in read.subfolders) + self._pending.extend(subfolders) + self.folders.extend(node for node, _ in subfolders) self.files += read.files self.size += read.size self.errors.extend(read.errors) @@ -393,6 +401,17 @@ def _finish(self, folder: Node, path: str, read: _FolderRead) -> None: _add_to_ancestors(folder, read) +def _unkept(subfolders: list[tuple[Node, str]], keep: frozenset[tuple[str, int, int]]) -> list[tuple[Node, str]]: + """The subfolders still to read; those ``keep`` names by name and identity are marked ``KEPT`` instead.""" + unread = [] + for child, path in subfolders: + if child.snapshot is not None and (child.name, *unpack_snapshot(child.snapshot).identity) in keep: + child.error = KEPT + else: + unread.append((child, path)) + return unread + + def _add_to_ancestors(folder: Node, read: _FolderRead) -> None: """Count what reading ``folder`` found into it and every folder above it (running totals).""" subfolders = len(read.subfolders) diff --git a/je_file_tree/core/windows_watch.py b/je_file_tree/core/windows_watch.py index 19a591ce..713ad72c 100644 --- a/je_file_tree/core/windows_watch.py +++ b/je_file_tree/core/windows_watch.py @@ -7,7 +7,9 @@ import struct import threading -from je_file_tree.core.change_watch import ChangeBatch, POLL_SECONDS, WatchFile, WatchFolder, changed, check_folder +from je_file_tree.core.change_watch import ( + POLL_SECONDS, ChangeBatch, Position, WatchFile, WatchFolder, changed, check_folder, +) from je_file_tree.core.compression_ops import _kernel from je_file_tree.core.windows_allocation import _FileID, _Identity from je_file_tree.core.windows_trash import _volume @@ -33,21 +35,19 @@ class _Overlapped(ctypes.Structure): ("offset", ctypes.c_uint32), ("offset_high", ctypes.c_uint32), ("event", ctypes.c_void_p)] -def _parent(root: str, name: str, scopes: set[str]) -> str: +def _parent(root: str, name: str) -> str: if (not name or ntpath.isabs(name) or ntpath.splitdrive(name)[0] or "\0" in name or any(part in ("", ".", "..") for part in name.replace("/", "\\").split("\\"))): raise ValueError("Unsafe Windows notification name") - parent = ntpath.dirname(ntpath.join(root, name)) - while parent not in scopes: - next_parent = ntpath.dirname(parent) - if parent in (next_parent, root): - return root - parent = next_parent - return parent + return ntpath.dirname(ntpath.join(root, name)) def parse_notifications(payload: bytes, root: str, scopes: set[str]) -> ChangeBatch: - """Validate native chained records and map names only to already captured folder parents.""" + """Validate native chained records; each names the folder inside ``root`` whose entries changed. + + That folder may be newer than the scan: the window reads the deepest folder it scanned there, which + lists the new one and reads it whole. ``scopes`` tells a scanned folder's own change from a file's. + """ if not payload: return changed(set(), full=True, reason="notification_lost") if len(payload) > _BUFFER: @@ -61,7 +61,7 @@ def parse_notifications(payload: bytes, root: str, scopes: set[str]) -> ChangeBa if action not in (1, 2, 3, 4, 5) or length % 2 or not length or end > len(payload): raise ValueError("Invalid Windows notification record") name = payload[offset + _NOTIFY.size:end].decode("utf-16-le") - parent = _parent(root, name, scopes) + parent = _parent(root, name) named = ntpath.normpath(ntpath.join(root, name)) paths.add(named if action == _ACTION_MODIFIED and named in scopes else parent) if not next_offset: @@ -148,7 +148,16 @@ def _journal(kernel, handle) -> tuple[int, int, int]: return identifier, max(first, lowest), next_usn -def _watch_usn(kernel, folders, on_change, cancel, ready, shared) -> None: +def _start(journal: int, first: int, next_usn: int, since: Position | None) -> int | None: + """Where to read from: on from ``since`` while the same journal still holds it, else None.""" + if since is None: + return next_usn + if since[0] == journal and first <= since[1] <= next_usn: + return since[1] + return None + + +def _watch_usn(kernel, folders, on_change, cancel, ready, shared, *, since=None, moved=None) -> None: # noqa: PLR0913 root, identifier = _volume(folders[0].path) scopes = {folder.inode: folder.path for folder in folders} aliases = {file.inode: file.parents for file in shared} @@ -158,7 +167,11 @@ def _watch_usn(kernel, folders, on_change, cancel, ready, shared) -> None: if handle == ctypes.c_void_p(-1).value: raise ctypes.WinError(ctypes.get_last_error()) try: - journal, _first, cursor = _journal(kernel, handle) + journal, first, cursor = _journal(kernel, handle) + cursor = _start(journal, first, cursor, since) + if cursor is None: + on_change(changed(set(), full=True, reason="journal_changed")) + return if ready: ready("usn") while not cancel.is_set(): @@ -171,6 +184,8 @@ def _watch_usn(kernel, folders, on_change, cancel, ready, shared) -> None: return payload = _control(kernel, handle, _READ_USN, _READ.pack(cursor, 0xFFFFFFFF, 0, 0, 0, journal)) cursor, batch = parse_usn(payload, scopes, folders[0].path, cursor, aliases) + if moved: + moved((journal, cursor)) if (batch.folders or batch.full) and not cancel.is_set(): on_change(batch) if batch.full or cancel.wait(POLL_SECONDS): @@ -267,9 +282,10 @@ def _watch_notifications(kernel, folders, on_change, cancel, ready) -> None: kernel.CloseHandle(handle) -def watch_windows(folders: tuple[WatchFolder, ...], on_change: Callable[[ChangeBatch], None], +def watch_windows(folders: tuple[WatchFolder, ...], on_change: Callable[[ChangeBatch], None], # noqa: PLR0913 cancel: threading.Event, ready: Callable[[str], None] | None, - shared: tuple[WatchFile, ...] = ()) -> None: + shared: tuple[WatchFile, ...] = (), *, since: Position | None = None, + moved: Callable[[Position], None] | None = None) -> None: """Use an existing journal when accessible; fallback is native, recursive, read-only and cancelable.""" kernel = _apis() announced = False @@ -281,7 +297,7 @@ def initialized(backend: str) -> None: ready(backend) try: - _watch_usn(kernel, folders, on_change, cancel, initialized, shared) + _watch_usn(kernel, folders, on_change, cancel, initialized, shared, since=since, moved=moved) except (OSError, ValueError, UnicodeError) as error: _LOG.debug("USN feed unavailable; using recursive notifications: %s", error) if not cancel.is_set(): diff --git a/je_file_tree/gui/change_watch.py b/je_file_tree/gui/change_watch.py index 3fb0977c..c0e26f09 100644 --- a/je_file_tree/gui/change_watch.py +++ b/je_file_tree/gui/change_watch.py @@ -1,18 +1,27 @@ -"""Default-off per-tab native following; bounded metadata events only request fresh scans.""" +"""Default-off per-tab native following; bounded metadata events only request fresh listings. + +A change reads again only the folder whose own entries changed (``core/refresh.py``): its subfolders +are kept, so a file saved directly in the scanned folder never reads the whole scan again. The native +feed keeps running while that folder is read; the scope is captured again afterwards and a new feed +takes over only once it is armed, an NTFS journal feed reading on from where the refresh began, so +nothing that happens meanwhile goes unseen. Only a lost or overflowing feed reads the whole root again. +""" from __future__ import annotations +import os import sys import threading import time +from collections.abc import Callable from typing import TYPE_CHECKING from PySide6.QtCore import QObject, QThread, Qt, QTimer from PySide6.QtWidgets import QApplication, QLabel -from je_file_tree.core.change_watch import MAX_FOLDERS, ChangeBatch, changed, watch -from je_file_tree.core.node import Node, outermost -from je_file_tree.core.pacing import give_way +from je_file_tree.core.change_watch import ChangeBatch, Position, WatchScope, captured_scope, merged, watch_scope +from je_file_tree.core.node import Node, folder_holding +from je_file_tree.core.scanner import EXCLUDED from je_file_tree.gui.i18n import tr from je_file_tree.gui.scan_worker import wait_for from je_file_tree.gui.worker_lifecycle import after_threads, queue_worker @@ -22,7 +31,6 @@ ENABLED_KEY = "follow_changes" COALESCE_SECONDS = 1.0 -RECONCILE_SECONDS = 300.0 def supported() -> bool: @@ -30,30 +38,44 @@ def supported() -> bool: return sys.platform == "win32" or sys.platform.startswith("linux") -def _inside(node: Node, ancestor: Node) -> bool: - while node is not ancestor and node.parent is not None: - node = node.parent - return node is ancestor - - -class ChangeWorker(QThread): - """Capture folder maps off the GUI thread and retain at most one bounded dirty batch.""" +class CaptureWorker(QThread): + """Read the tree once for a feed's scope. It reads the tree, so scans and operations wait for it.""" def __init__(self, root: Node, parent: QObject) -> None: super().__init__(parent) self.root = root self.cancel = threading.Event() - self.folders: dict[str, Node] = {} + self.scope: WatchScope | None = None + self.error = "" + + def run(self) -> None: + """Capture off the GUI thread; errors stay visible on the controller.""" + try: + self.scope = captured_scope(self.root, self.cancel) + except (OSError, ValueError, UnicodeError) as error: + self.error = str(error) + + +class ChangeWorker(QThread): + """One native feed over a captured scope, retaining at most one bounded dirty batch. + + It never reads the tree, so a folder is read again while it runs. That is why it has no Qt parent: + ``queue_worker`` waits for every thread a window holds. Its controller keeps it until it joins. + """ + + def __init__(self, scope: WatchScope, since: Position | None = None) -> None: + super().__init__() + self.scope, self.since = scope, since + self.cancel = threading.Event() self.backend, self.error = "", "" + self.position: Position | None = None self._lock = threading.Lock() self._batch = ChangeBatch() def record(self, batch: ChangeBatch) -> None: """Coalesce at the producer; event storms cannot create an unbounded Qt signal queue.""" with self._lock: - self._batch = changed(set(self._batch.folders) | set(batch.folders), - full=self._batch.full or batch.full, - reason=batch.reason or self._batch.reason) + self._batch = merged(self._batch, batch) def take(self) -> ChangeBatch: """Transfer pending metadata once under the producer lock.""" @@ -64,37 +86,62 @@ def take(self) -> ChangeBatch: def run(self) -> None: """Retain native setup/errors until the owning controller cancels and joins this worker.""" try: - for node in self.root.iter_nodes(): - if self.cancel.is_set(): - return - if node.is_dir: - give_way() - if not node.is_link and node.path is not None: - self.folders[node.path] = node - if len(self.folders) > MAX_FOLDERS: - raise ValueError("Follow changes exceeds the captured-folder limit") - watch(self.root, self.record, self.cancel, ready=self._initialized) + watch_scope(self.scope, self.record, self.cancel, ready=self._initialized, since=self.since, + moved=self._moved) except (OSError, ValueError, UnicodeError) as error: self.error = str(error) def _initialized(self, backend: str) -> None: self.backend = backend + def _moved(self, position: Position) -> None: + self.position = position + + +def targets(root: Node, batch: ChangeBatch) -> list[tuple[Node, bool]]: + """The scanned folders to read again for ``batch``, each with whether to read it whole. + + A reported path maps to the deepest folder scanned there; one below it is new, and that folder's + listing reads it whole. Folders inside one read whole, and excluded folders, are left out. + """ + found: dict[Node, bool] = {} + for path, whole in (*((path, False) for path in batch.folders), *((path, True) for path in batch.trees)): + node = folder_holding(root, path) + if node is None or node.error == EXCLUDED or node.path is None: + continue + exact = os.path.normcase(node.path) == os.path.normcase(os.path.normpath(path)) + found[node] = found.get(node, False) or (whole and exact) + whole_ones = {node for node, whole in found.items() if whole} + + def covered(node: Node) -> bool: + above = node.parent + while above is not None: + if above in whole_ones: + return True + above = above.parent + return False + + chosen = [(node, whole) for node, whole in found.items() if not covered(node)] + return sorted(chosen, key=lambda item: item[0].depth) + class FollowChanges(QObject): - """Own one tab's monitor, defer busy work and periodically reconcile observational gaps.""" + """Own one tab's feed, defer busy work and hand over between feeds without a gap.""" def __init__(self, window: MainWindow) -> None: super().__init__(window) self.window = window self.worker: ChangeWorker | None = None + self._previous: ChangeWorker | None = None # the feed before ``worker``, until that one is armed + self._capture: CaptureWorker | None = None self.root: Node | None = None self.enabled, self._closing = False, False self._pending = ChangeBatch() - self._operation_gap = False # the reader was stopped for a source operation: read the whole root again - self._since, self._reconcile = 0.0, 0.0 + self._operation_gap = False # the feed was stopped for a source operation: read the whole root again + self._handoff: Position | None = None # where the next journal feed reads on from + self._since = 0.0 self._status, self._detail = "follow_waiting", "" - self._retiring: set[ChangeWorker] = set() + self._retiring: set[QThread] = set() self.label = QLabel(window) self.label.setTextFormat(Qt.TextFormat.PlainText) self.label.setMaximumWidth(300) @@ -126,9 +173,9 @@ def configure(self, enabled: bool) -> None: self.timer.stop() def adopt(self, root: Node) -> None: - """Rearm after a complete/branch result; pending captured paths survive branch replacement.""" - previous = self.root - self.stop() + """Follow ``root``: capture its scope again; a feed already following it reports on meanwhile.""" + if root is not self.root or self.worker is None or self.worker.error: + self.stop() self.root = root if not self.enabled or self._closing: return @@ -137,46 +184,109 @@ def adopt(self, root: Node) -> None: return outcome = self.window.results.outcome if root.path is None or outcome is None or outcome.partial: + self.stop() self._set_status("follow_incomplete") return - self._set_status("follow_starting") - worker = ChangeWorker(root, self) - self.worker = worker - queue_worker(worker, lambda: self.worker is worker and not self._closing - and not self.window._closing and not worker.cancel.is_set(), self) - if root is not previous or not self._reconcile: - self._reconcile = time.monotonic() + RECONCILE_SECONDS - - def stop(self, *, clear: bool = False, wait: bool = False) -> tuple[ChangeWorker, ...]: - """Join through the GUI's shared gate; retain the final bounded batch before releasing maps.""" - worker, self.worker = self.worker, None - if worker is not None: - worker.cancel.set() - root = self.root - self._retiring.add(worker) - after_threads((worker,), lambda: self._retired(worker, root, clear), self) + if self._capture is not None: + self._retire(self._capture) + if self.worker is None: + self._set_status("follow_starting") + capture = CaptureWorker(root, self) + self._capture = capture + capture.finished.connect(lambda: self._captured(capture)) + queue_worker(capture, lambda: self._capture is capture and not self._closing + and not self.window._closing and not capture.cancel.is_set(), self) + + def _captured(self, capture: CaptureWorker) -> None: + if capture is not self._capture: + return + self._capture = None + cancelled = capture.cancel.is_set() + self._retire(capture) # joined already: this only releases it + if cancelled or self._closing or not self.enabled or capture.root is not self.root: + return + if capture.error: + self.stop() + self._set_status("follow_failed", capture.error) + return + if capture.scope is not None and capture.scope.folders: + self._arm(capture.scope) + + def _arm(self, scope: WatchScope) -> None: + """Start a feed over ``scope``: beside the running one, which it then replaces, else after the old ones join.""" + if self._previous is not None: + self._retire(self._previous) + feed = ChangeWorker(scope, self._handoff) + self._previous, self.worker = self.worker, feed + self._handoff = None + if self._previous is not None: + feed.start() + return + stopped = tuple(worker for worker in self._retiring if isinstance(worker, ChangeWorker)) + queue_worker(feed, lambda: self.worker is feed and not self._closing and not feed.cancel.is_set(), self, + readers=stopped) + + def refreshed(self, root: Node, *, structural: bool) -> None: + """Folders of ``root`` were read again: capture the scope again if their subfolders changed. + + Otherwise the running feed's scope still holds every folder, and it saw everything since the + refresh began, so it simply carries on. + """ + if structural or self._capture is not None or self.worker is None or self.root is not root: + self.adopt(root) + else: + self._handoff = None + + def hold(self) -> None: + """A folder is about to be read again: the next journal feed reads on from here.""" + if self._handoff is None: + self._handoff = self._position() + + def _position(self) -> Position | None: + known = [feed.position for feed in (self._previous, self.worker) if feed is not None and feed.position] + return min(known) if known else None + + def _retire(self, worker: CaptureWorker | ChangeWorker, then: Callable[[], None] | None = None) -> None: + """Cancel ``worker`` and keep it until it joins; a feed's last batch is kept for the same root.""" + worker.cancel.set() + root = self.root + self._retiring.add(worker) + + def joined() -> None: + self._retiring.discard(worker) + if isinstance(worker, ChangeWorker) and not self._closing and self.root is root and root is not None: + self._merge(worker.take()) + worker.deleteLater() + if then is not None: + then() + + after_threads((worker,), joined, self) + + def stop(self, *, clear: bool = False, wait: bool = False) -> tuple[QThread, ...]: + """Cancel the capture and every feed; join through the GUI's shared gate unless ``wait``.""" + for worker in (self._capture, self._previous, self.worker): + if worker is not None: + self._retire(worker) + self._capture = self._previous = self.worker = None if clear: self.root = None self._pending, self._since = ChangeBatch(), 0.0 self._operation_gap = False - self._reconcile = 0.0 + self._handoff = None retiring = tuple(self._retiring) if wait: for pending in retiring: wait_for(pending) return retiring - def _retired(self, worker: ChangeWorker, root: Node | None, clear: bool) -> None: - self._retiring.discard(worker) - if not clear and not self._closing and self.root is root: - self._merge(worker.take()) - if worker.error: - self._set_status("follow_failed", worker.error) - worker.deleteLater() + def quiesce(self, *, wait: bool = True) -> tuple[QThread, ...]: + """Stop tree capture before source operations; their observation gap requires full reconciliation. - def quiesce(self, *, wait: bool = True) -> tuple[ChangeWorker, ...]: - """Stop tree capture before source operations; their observation gap requires full reconciliation.""" + A journal feed's position is kept: the next one reads on from it, which closes that gap. + """ active = self.worker is not None + if active: + self.hold() retiring = self.stop(wait=wait) if active: if not self._operation_gap and not self._pending.full and not self._pending.folders: @@ -187,18 +297,24 @@ def quiesce(self, *, wait: bool = True) -> tuple[ChangeWorker, ...]: def resume(self) -> None: """Follow again after an operation whose changes the results already show or have on offer. - Stopping the reader for a source operation asks for the whole root to be read again, because - nothing was observed meanwhile. A move to the Recycle Bin or a permanent delete takes its - entries out of the results itself (and leaves the folders of anything else on offer), so only - that request is withdrawn: folders reported before the stop stay pending, and a whole-root - request made for any other reason stays. What another program changed while the reader was - stopped is found by the periodic reconciliation. + Stopping the feed for a source operation asks for the whole root to be read again, because + nothing was observed meanwhile, unless a journal feed reads on from where it stopped. A move to + the Recycle Bin or a permanent delete takes its entries out of the results itself (and leaves + the folders of anything else on offer), so only that request is withdrawn: folders reported + before the stop stay pending, and a whole-root request made for any other reason stays. """ self._operation_gap = False root = self.root if self.enabled and not self._closing and root is not None and self.worker is None: self.adopt(root) + def request(self, path: str | None, *, whole: bool = False) -> None: + """Read ``path`` again later (the whole root for None), as if its feed had reported it.""" + if path is None: + self._merge(ChangeBatch(full=True, reason="root_changed")) + else: + self._merge(ChangeBatch(trees=(path,)) if whole else ChangeBatch((path,))) + def shutdown(self, *, wait: bool = True) -> None: """Prevent late replies and join all native work before the tab is destroyed.""" self._closing = True @@ -206,33 +322,53 @@ def shutdown(self, *, wait: bool = True) -> None: self.stop(clear=True, wait=wait) def _merge(self, batch: ChangeBatch) -> None: - if not batch.folders and not batch.full: + if not batch.folders and not batch.trees and not batch.full: return - if not self._pending.full and not self._pending.folders: + pending = self._pending + if not pending.full and not pending.folders and not pending.trees: self._since = time.monotonic() - self._pending = changed(set(self._pending.folders) | set(batch.folders), - full=self._pending.full or batch.full, - reason=batch.reason or self._pending.reason) + self._pending = merged(pending, batch) def tick(self) -> None: """Poll bounded events once a second and avoid scans during reviews, Undo offers or analysis.""" if not self.enabled or self._closing or self.window._closing: return + for feed in (self._previous, self.worker): + if feed is not None: + self._merge(feed.take()) worker = self.worker if worker is not None: - self._merge(worker.take()) - if worker.error: + if worker.error and self._previous is not None and not self._previous.error: + self._arm_alone(worker) + elif worker.error: self._set_status("follow_failed", worker.error) return elif worker.backend: + if self._previous is not None: + self._retire(self._previous) + self._previous = None + if worker.since is not None and worker.backend == "usn": + self._operation_gap = False # it read on from where the stopped feed was self._set_status("follow_active", worker.backend) - if (self.root is not None and worker is not None and not worker.error and worker.backend - and time.monotonic() >= self._reconcile): - self._merge(ChangeBatch(full=True, reason="periodic_reconciliation")) if self.window._trash_rescans: self.window._rescan_after_trash() # folders left on offer before following was turned on self._flush() + def _arm_alone(self, failed: ChangeWorker) -> None: + """The new feed could not run beside the old one (an inotify watch limit): stop that one first.""" + previous, root = self._previous, self.root + self._previous = self.worker = None + self._retire(failed) + if previous is None: + return + self.hold() + + def rearm() -> None: + if self.enabled and not self._closing and self.root is root and self.worker is None: + self._arm(failed.scope) + + self._retire(previous, rearm) + def _flush(self) -> None: window = self.window if (window._worker is not None or window.operation_busy or window._analysers @@ -244,26 +380,18 @@ def _flush(self) -> None: or outcome.partial or outcome.result.root is not self.root): return batch = self._pending - full = batch.full or self._operation_gap - if not full and not batch.folders: - return - if full: - node = self.root + if batch.full or self._operation_gap: self._pending, self._operation_gap = ChangeBatch(), False - else: - worker = self.worker - if worker is None or not worker.backend or worker.error: - return - candidates = [worker.folders.get(path) for path in batch.folders] - if any(node is None or not node.is_in(self.root) for node in candidates): - self._merge(ChangeBatch(full=True, reason="scope_changed")) - return - node = outermost(candidates)[0] - # Keep unrelated dirty branches; the next result's worker supplies their fresh map. - remaining = {path for path, candidate in zip(batch.folders, candidates, strict=True) - if not _inside(candidate, node)} - self._pending = changed(remaining) - window.rescan_folder(node) + window.rescan_folder(self.root) + return + worker = self.worker + if not batch.folders and not batch.trees or worker is None or not worker.backend or worker.error: + return + self._pending = ChangeBatch() + chosen = targets(self.root, batch) + if chosen: + self.hold() + window.refresh_folders(chosen) def _set_status(self, key: str, detail: str = "") -> None: self._status, self._detail = key, detail diff --git a/je_file_tree/gui/folder_refresh.py b/je_file_tree/gui/folder_refresh.py new file mode 100644 index 00000000..001d4fd5 --- /dev/null +++ b/je_file_tree/gui/folder_refresh.py @@ -0,0 +1,139 @@ +"""Read again only the folders Follow changes found changed, and put them in place in the results. + +Each folder's own entries are listed again and its subfolders are kept (``core/refresh.py``), so a +change directly in the scanned folder reads that one folder, not the whole scan. A folder reported +whole (too many changes below it) is read entirely. The worker stands in for a scan worker: while it +runs, everything that waits for a scan waits for it. +""" + +from __future__ import annotations + +from dataclasses import replace +from typing import TYPE_CHECKING + +from PySide6.QtCore import QObject + +from je_file_tree.core.formatting import format_count +from je_file_tree.core.node import Node +from je_file_tree.core.refresh import kept_subfolders +from je_file_tree.core.scanner import ScanCancelledError, ScanOptions, ScanResult, scan +from je_file_tree.core.snapshot import unpack_snapshot +from je_file_tree.gui.i18n import tr +from je_file_tree.gui.scan_worker import ScanWorker +from je_file_tree.gui.worker_lifecycle import after_threads, queue_worker + +if TYPE_CHECKING: + from je_file_tree.gui.main_window import MainWindow + +STATUS_TIMEOUT_MS = 8000 + +Plan = tuple[str, frozenset[tuple[str, int, int]]] # a folder and the subfolders its listing keeps + + +class ListingWorker(ScanWorker): + """Lists each planned folder; ``succeeded`` carries one ``ScanResult`` per plan, None for a folder gone.""" + + def __init__(self, plans: tuple[Plan, ...], options: ScanOptions, parent: QObject | None = None) -> None: + super().__init__(plans[0][0], options, parent) + self.plans = plans + self.linked = False # a file listed has another name: the feed's scope must be captured again + + def _run_scan(self) -> None: + results: list[ScanResult | None] = [] + for path, keep in self.plans: + if self._cancel.is_set(): + self.cancelled.emit(None) + return + try: + result = scan(path, options=replace(self._options, keep=keep), cancel=self._cancel, pause=self._pause) + self.linked = self.linked or _has_links(result.root) + results.append(result) + except NotADirectoryError: + results.append(None) # removed meanwhile: the folder holding it shows that + except ScanCancelledError: + self.cancelled.emit(None) + return + self._scanning = False + self.succeeded.emit(tuple(results)) + + +def _has_links(folder: Node) -> bool: + return any(not child.is_dir and child.snapshot is not None and unpack_snapshot(child.snapshot).links > 1 + for child in folder.iter_nodes() if child is not folder) + + +def _subfolders(node: Node) -> set[tuple[str, tuple[int, int] | None]]: + return {(child.name, None if child.snapshot is None else unpack_snapshot(child.snapshot).identity) + for child in node.children if child.is_dir} + + +def refresh_folders(window: MainWindow, chosen: list[tuple[Node, bool]]) -> None: + """Read ``chosen`` again on a worker: each folder's own entries, or all of it when its flag is set.""" + outcome = window.results.outcome + if window._worker is not None or window.operation_busy or outcome is None or not chosen: + return + root = outcome.result.root + if ((outcome.result.hard_links is not None) != window._actions["count_hard_links"].isChecked() + or any(whole and node is root for node, whole in chosen)): + window.rescan_folder(root) # only a whole scan counts every hard link again + return + paused = window._background_pause(wait=False) if window._background_pause is not None else () + window._undo.expire() + window._analyser = None + plans = tuple((node.path, frozenset() if whole else kept_subfolders(node)) for node, whole in chosen) + # Listed, never taken from the file table, which can be a few seconds behind; hard links are counted + # again over the whole tree afterwards, as after a folder rescan. + options = replace(window._scan_options(), mft=False, count_hard_links=False) + worker = ListingWorker(plans, options, window) + before = [_subfolders(node) for node, _whole in chosen] + worker.succeeded.connect(lambda results: window._is_current(worker) and _refreshed( + window, chosen, results, before=before, linked=worker.linked)) + worker.failed.connect(lambda reason: window._is_current(worker) and _ended(window, tr("follow_refresh_failed", + reason=reason))) + worker.cancelled.connect(lambda _outcome: window._is_current(worker) and _ended(window, tr("scan_cancelled"))) + worker.finished.connect(lambda: after_threads((worker,), lambda: window._release_scan(worker), window)) + window._worker = worker + window.scan_activity_changed.emit() + window._scan_workers.add(worker) + window._update_actions() + queue_worker(worker, lambda: not window._closing and window._worker is worker and not worker._cancel.is_set(), + window, lambda: _discarded(window, worker), readers=paused) + + +def _refreshed(window: MainWindow, chosen: list[tuple[Node, bool]], results: tuple[ScanResult | None, ...], *, + before: list[set[tuple[str, tuple[int, int] | None]]], linked: bool) -> None: + window._scan_ended() + listings = [(node, result) for (node, _whole), result in zip(chosen, results, strict=True) if result is not None] + follow = window._follow + for (node, _whole), result in zip(chosen, results, strict=True): + if result is None: + follow.request(node.parent.path if node.parent is not None else None) + failed = window.results.refill_folders(listings) + for node in failed: + follow.request(node.path, whole=True) # a kept subfolder was replaced meanwhile: read it all + # The feed's scope holds every folder and every file with another name: capture it again only when + # a listing found, lost or replaced a subfolder, or listed such a file. Otherwise the feed carries on. + structural = linked or bool(failed) or len(listings) != len(chosen) or any( + whole or _subfolders(node) != folders for (node, whole), folders in zip(chosen, before, strict=True)) + window._update_actions() + root = window.results.tree_model.root + if root is not None and window.results.outcome is not None: + window._start_analysis(root, with_capacity=True, count=window.results.outcome.result.hard_links is not None) + follow.refreshed(root, structural=structural) + window.statusBar().showMessage(tr("follow_refreshed", count=format_count(len(listings))), STATUS_TIMEOUT_MS) + + +def _ended(window: MainWindow, message: str) -> None: + window._scan_ended() + window._update_actions() + window.statusBar().showMessage(message, STATUS_TIMEOUT_MS) + + +def _discarded(window: MainWindow, worker: ListingWorker) -> None: + if worker is window._worker: + if window._closing: + window._worker = None + window.scan_activity_changed.emit() + else: + _ended(window, tr("scan_cancelled")) + window._release_scan(worker) diff --git a/je_file_tree/gui/main_window.py b/je_file_tree/gui/main_window.py index 060b2c11..cc12f8f0 100644 --- a/je_file_tree/gui/main_window.py +++ b/je_file_tree/gui/main_window.py @@ -43,7 +43,7 @@ from je_file_tree.core.scanner import DEFAULT_WORKERS, ScanOptions from je_file_tree.core.system_files import system_file from je_file_tree.core.trash_size import TrashUsage -from je_file_tree.gui import elevation, file_actions, shell_integration +from je_file_tree.gui import elevation, file_actions, folder_refresh, shell_integration from je_file_tree.gui.locale_font import apply_language_font from je_file_tree.gui.shell_dialog import ShellIntegrationDialog from je_file_tree.gui.elevation_dialog import ElevationDialog @@ -726,12 +726,10 @@ def rescan_folder(self, node: Node, *, exact_allocation: bool = False) -> None: """Scan one folder again and swap it into the results (the whole scan when it is the root).""" if self._worker is not None or self.operation_busy or self.results.outcome is None: return - # The scan waits for the window's readers. Follow changes is stopped as a re-arm, keeping what it - # reported: stopped as for a source operation it read the whole root again after every folder it - # refreshed. Its short gap is covered by the periodic reconciliation, like any re-arm. - if self._background_pause is not None: - self._background_pause(wait=False) - self._follow.stop() + # The scan waits for the window's readers, except Follow changes: its feed reads nothing of the tree + # and keeps reporting while the folder is read, until the feed armed after it takes over. + paused = self._background_pause(wait=False) if self._background_pause is not None else () + self._follow.hold() self._undo.expire() if self.results.outcome.result.root.path is None: roots = tuple(child.path for child in self.results.outcome.result.root.children) @@ -767,7 +765,11 @@ def rescan_folder(self, node: Node, *, exact_allocation: bool = False) -> None: self.results.scan_bar.start() self._update_actions() queue_worker(worker, lambda: not self._closing and self._worker is worker and not worker._cancel.is_set(), - self, lambda: self._discard_branch(worker)) + self, lambda: self._discard_branch(worker), readers=paused) + + def refresh_folders(self, chosen: list[tuple[Node, bool]]) -> None: + """Read again only what Follow changes found changed (``gui/folder_refresh.py``).""" + folder_refresh.refresh_folders(self, chosen) def _discard_branch(self, worker: ScanWorker) -> None: if worker is self._worker: @@ -779,7 +781,6 @@ def _discard_branch(self, worker: ScanWorker) -> None: self._release_scan(worker) def _branch_rescanned(self, old: Node, outcome: ScanOutcome, before: int) -> None: - self._follow.stop() self._scan_ended() self.results.scan_bar.hide() new = self.results.replace_branch(old, outcome.result) diff --git a/je_file_tree/gui/results_view.py b/je_file_tree/gui/results_view.py index 20087a85..2b135109 100644 --- a/je_file_tree/gui/results_view.py +++ b/je_file_tree/gui/results_view.py @@ -60,6 +60,7 @@ from je_file_tree.core.hard_links import HardLinkPlan from je_file_tree.core.node import Node from je_file_tree.core.protected import protected_places +from je_file_tree.core.refresh import refreshed_errors from je_file_tree.gui.node_text import node_path from je_file_tree.core.compare import SavedScan from je_file_tree.core.scanner import ScanProgress, ScanResult @@ -363,6 +364,40 @@ def replace_branch(self, old: Node, fresh: ScanResult) -> Node: self.selection_changed.emit(self.selected_node()) return new + def refill_folders(self, listings: list[tuple[Node, ScanResult]]) -> list[Node]: + """Put fresh listings of changed folders in place; returns the folders that could not take theirs. + + Each folder stays the same node with the subfolders it kept (``core/refresh.py``), so the view + keeps its place, expanded folders and selection there. The rest follows as after ``replace_branch``. + """ + self._stop_focused() + self.capacity.set_ledger(None) + self.tree_model.set_drive_total(None) + view_root = self.charts.view_root + holder = next((node for node, _ in reversed(listings) + if view_root is not None and _is_within(view_root, node)), None) + done = self.tree_model.refill(listings) + if self._outcome is not None: + errors = self._outcome.result.errors + for folder, fresh, kept in done: + errors[:] = refreshed_errors(errors, folder.path, kept, fresh.errors) + self._outcome = dataclasses.replace(self._outcome, capacity=None, top_folders=None) + self.problems_model.set_rows(errors) + root = self.tree_model.root + if view_root is not None and root is not None and not view_root.is_in(root): + view_root = holder + self.charts.set_view_root(view_root) + self.charts.invalidate() + self.search.rerun() + self.duplicates.prune() + self.cleanup.refresh() + self.users.refresh() + self.changes.refresh() + self._update_texts() + self.selection_changed.emit(self.selected_node()) + refilled = {id(folder) for folder, _, _ in done} + return [node for node, _ in listings if id(node) not in refilled and root is not None and node.is_in(root)] + def apply_counted(self, plan: HardLinkPlan) -> None: """Show counted totals planned for the whole tree after a branch rescan (``plan_hard_links``).""" outcome = self._outcome diff --git a/je_file_tree/gui/strings.py b/je_file_tree/gui/strings.py index 129bdaaa..166cc57c 100644 --- a/je_file_tree/gui/strings.py +++ b/je_file_tree/gui/strings.py @@ -375,8 +375,10 @@ def _cjk(html: str) -> str: 'background_warning': '{detail}', 'background_low_space': 'Low available space on {root}: {free} ({percent}%)', 'action_follow_changes': 'Follow changes', - 'action_follow_changes_tip': ('Off by default. Refresh changed folders on Windows/Linux; ' - 'reconcile the root every five minutes.'), + 'action_follow_changes_tip': ('Off by default. On Windows/Linux, read again only the folders that ' + 'changed, keeping their unchanged subfolders.'), + 'follow_refreshed': 'Updated {count} changed folders', + 'follow_refresh_failed': 'Could not update the changed folders: {reason}', 'follow_waiting': 'Follow changes: waiting for a scan', 'follow_starting': 'Follow changes: starting…', 'follow_active': 'Following changes ({detail})', @@ -2219,7 +2221,9 @@ def _cjk(html: str) -> str: 'background_warning': '{detail}', 'background_low_space': '{root} 可用空間不足:{free}({percent}%)', 'action_follow_changes': '跟隨變更', - 'action_follow_changes_tip': '預設關閉。在 Windows/Linux 更新變動資料夾,每五分鐘完整核對根目錄。', + 'action_follow_changes_tip': '預設關閉。在 Windows/Linux 只重新讀取有變動的資料夾,保留其中未變動的子資料夾。', + 'follow_refreshed': '已更新 {count} 個變動的資料夾', + 'follow_refresh_failed': '無法更新變動的資料夾:{reason}', 'follow_waiting': '跟隨變更:等待掃描', 'follow_starting': '跟隨變更:啟動中…', 'follow_active': '跟隨變更({detail})', @@ -3922,7 +3926,9 @@ def _cjk(html: str) -> str: 'background_warning': '{detail}', 'background_low_space': '{root} 可用空间不足:{free}({percent}%)', 'action_follow_changes': '跟随变更', - 'action_follow_changes_tip': '默认关闭。在 Windows/Linux 更新变化文件夹,每五分钟完整核对根目录。', + 'action_follow_changes_tip': '默认关闭。在 Windows/Linux 只重新读取有变化的文件夹,保留其中未变化的子文件夹。', + 'follow_refreshed': '已更新 {count} 个变化的文件夹', + 'follow_refresh_failed': '无法更新变化的文件夹:{reason}', 'follow_waiting': '跟随变更:等待扫描', 'follow_starting': '跟随变更:启动中…', 'follow_active': '跟随变更({detail})', @@ -5701,9 +5707,11 @@ def _cjk(html: str) -> str: 'background_low_space': '{root} の空き容量が少ない: {free} ({percent}%)', 'action_follow_changes': '変更を追跡する', 'action_follow_changes_tip': ( - 'デフォルトではオフです。 Windows/Linux で変更されたフォルダーを更新します。 5分ごとにルートを調整し' - 'ます。' + 'デフォルトではオフです。 Windows/Linux で変更されたフォルダーだけを読み直し、変更のないサブフォルダーは' + 'そのまま残します。' ), + 'follow_refreshed': '変更された {count} 個のフォルダーを更新しました', + 'follow_refresh_failed': '変更されたフォルダーを更新できませんでした: {reason}', 'follow_waiting': '変更の追跡: スキャンを待機しています', 'follow_starting': '変更を追跡: 開始中…', 'follow_active': '次の変更点 ({detail})', @@ -7802,8 +7810,10 @@ def _cjk(html: str) -> str: 'background_low_space': '{root}의 사용 가능한 공간 부족: {free} ({percent}%)', 'action_follow_changes': '변경 사항 따르기', 'action_follow_changes_tip': ( - '기본적으로 꺼져 있습니다. Windows/Linux에서 변경된 폴더를 새로 고칩니다. 5분마다 루트를 조정합니다.' + '기본적으로 꺼져 있습니다. Windows/Linux에서 변경된 폴더만 다시 읽고, 변경되지 않은 하위 폴더는 그대로 둡니다.' ), + 'follow_refreshed': '변경된 폴더 {count}개를 업데이트했습니다', + 'follow_refresh_failed': '변경된 폴더를 업데이트할 수 없습니다: {reason}', 'follow_waiting': '변경 사항 따르기: 스캔 대기 중', 'follow_starting': '변경사항 따르기: 시작 중…', 'follow_active': '다음 변경 사항({detail})', diff --git a/je_file_tree/gui/tree_model.py b/je_file_tree/gui/tree_model.py index 74458025..c64dd182 100644 --- a/je_file_tree/gui/tree_model.py +++ b/je_file_tree/gui/tree_model.py @@ -39,8 +39,9 @@ from je_file_tree.core.archives import VirtualEntry from je_file_tree.core.hard_links import HardLinkPlan from je_file_tree.core.node import Node, detach_many +from je_file_tree.core.refresh import refill as refill_folder from je_file_tree.gui.node_text import node_name, node_path -from je_file_tree.core.scanner import EXCLUDED, NOT_SCANNED +from je_file_tree.core.scanner import EXCLUDED, NOT_SCANNED, ScanResult from je_file_tree.core.snapshot import unpack_snapshot from je_file_tree.gui.i18n import tr from je_file_tree.gui.list_transfer import PLACEHOLDER_ROLE @@ -645,6 +646,27 @@ def change() -> None: self._invalidate_size_order(parent) self._relayout(change) + def refill(self, listings: list[tuple[Node, ScanResult]]) -> list[tuple[Node, ScanResult, frozenset[str]]]: + """Put fresh listings of folders in place (``core.refresh.refill``), keeping what the view still can. + + Unlike ``replace``, each folder stays the same node, so it stays expanded and selected, and so do + the subfolders it kept. Returns each folder refilled, its listing and the names it kept; a folder + no longer in the tree, or whose kept subfolder was replaced, is left as it was. + """ + done: list[tuple[Node, ScanResult, frozenset[str]]] = [] + + def change() -> None: + self.invalidate_archives() + for folder, fresh in listings: + if self._root is None or not folder.is_in(self._root): + continue + kept = refill_folder(folder, fresh.root) + if kept is not None: + done.append((folder, fresh, kept)) + self._invalidate_size_order(folder) + self._relayout(change) + return done + def _index_at(self, node: Node | None, column: int) -> QModelIndex: if node is None: return QModelIndex() diff --git a/je_file_tree/gui/worker_lifecycle.py b/je_file_tree/gui/worker_lifecycle.py index 92680cf1..0e00527a 100644 --- a/je_file_tree/gui/worker_lifecycle.py +++ b/je_file_tree/gui/worker_lifecycle.py @@ -49,10 +49,17 @@ def retire_worker(worker: QThread, parent: QObject) -> None: def queue_worker(worker: QThread, current: Callable[[], bool], parent: QObject, - discarded: Callable[[], None] | None = None) -> None: - """Coalesce replacements behind retained workers; canceled/stale requests never start.""" - provider = getattr(parent, "pending_source_workers", None) - sources = () if provider is None else provider() + discarded: Callable[[], None] | None = None, *, readers: Iterable[QThread] | None = None) -> None: + """Coalesce replacements behind retained workers; canceled/stale requests never start. + + ``readers`` stands in for the parent's source readers (``pending_source_workers``, which stops them): + a scan that changes nothing on disk passes the ones it stopped itself and leaves the others running. + """ + if readers is None: + provider = getattr(parent, "pending_source_workers", None) + sources = () if provider is None else provider() + else: + sources = tuple(readers) previous = (pending for pending in (*parent.findChildren(QThread), *sources) if pending is not worker) def dispatch() -> None: if current(): diff --git a/progress.md b/progress.md index 82c03940..cd8559d3 100644 --- a/progress.md +++ b/progress.md @@ -23,6 +23,7 @@ ### 掃描 - **#142**(第二優先)[待驗證] 由主檔案表讀取 NTFS 磁碟的掃描(U-20261115-05、-07)只在一台機器的一顆系統磁碟上量測與比對。尚未驗證:檔案紀錄為 4096 位元組的磁碟(4Kn);檔案表本身高度分散的磁碟;資料磁碟與外接磁碟;啟用系統壓縮(CompactOS)或含大量雲端預留位置的磁碟;真正的 Windows 10;冷快取下與列出資料夾的秒數比較;CI 的 Windows 工作能否讀取檔案表(讀不到時 `test_mft_scan.py` 的實機測試會略過);已發行的 wheel 與獨立版是否含新版模組;有日光節約時間的時區下原生時間文字與 Python 是否相同(U-20261115-10)。另有 85 個資料夾的大小與 Windows 回報不同,原因未查明;這些項目在操作前會被判定為已變更而拒絕,重新掃描該資料夾即可。 +- **#144**(第二優先)[待驗證] 「跟隨變更」只重新讀取變動資料夾(U-20261120-01),目前只以 Windows 目錄通知(非系統管理員)與測試替身驗證。尚未驗證:以系統管理員執行時,NTFS USN 監看能否從保留的位置接著讀,交接期間不漏掉變動;Linux inotify 新舊監看短暫並存時,是否會超過監看數量上限;在使用者實際使用、變動頻繁的資料夾上,重新讀取監看範圍的背景負擔。 - **#138**(第三優先)[待驗證] 選用的原生掃描模組 `je_file_tree._native` 已隨 0.2.9 發行(PyPI 的 Windows wheel 與 Windows 獨立版),發行本身已驗證。尚未驗證的環境:真正的 Windows 10;未安裝 Visual C++ 可轉散發套件的乾淨機器(模組須可載入並被採用);非 NTFS 磁碟區中的 exFAT 與 ReFS(FAT32 已驗證),須確認掃描起點比對會正確決定是否使用原生讀取;ARM64 的 Windows(目前只發行 x64 wheel)。目前沒有這些環境。任何一項失敗時,程式仍以 Python 讀取掃描,結果相同。 - **#35**(第二優先)[受阻] 須在實際緩慢的遠端 SMB 連線上,量測 UNC 路徑與對應磁碟機在不同背景工作數量下的處理量,再建議該環境的並行程度。使用者目前未提供遠端共用資料夾或慢速連線驗證環境,本機回送連線的原生量測無法證明遠端效能。 - **#75**(第一優先)[受阻] 須透過原生目錄走訪防護,驗證 macOS 與其他 POSIX 系統上同一裝置內的掛載邊界。使用者目前未提供可拋棄的對應驗證環境。FreeBSD 的 nullfs 具有不同的 `st_dev`,無法證明此情境。未掛載的 APFS 同容器磁碟區容量仍未知。圖形介面與命令列的容量帳目仍為估計值;無法獨立量測的共用空間、保留空間、檔案系統中繼資料、遺漏位元組及其他磁碟區總量,均須維持未知。直接顯示垃圾桶標籤與 Finder 清空功能仍由 #21 追蹤。 diff --git a/test/test_change_watch.py b/test/test_change_watch.py index a20277f9..f7a697c5 100644 --- a/test/test_change_watch.py +++ b/test/test_change_watch.py @@ -87,7 +87,7 @@ def control(_kernel, _handle, code, _data=b""): def test_initial_usn_denial_falls_back_without_triggering_repeated_full_rescans(monkeypatch): batches, backends = [], [] - def denied(*_args): + def denied(*_args, **_kwargs): raise PermissionError("owned denied journal") def fallback(_kernel, _folders, _changed, _cancel, ready): @@ -106,14 +106,51 @@ def fallback(_kernel, _folders, _changed, _cancel, ready): assert not batches and backends == ["directory_changes"] -def test_windows_notification_parents_are_known_and_new_descendants_use_known_ancestor(): +def test_windows_notifications_name_the_folder_whose_entries_changed_even_when_it_is_new(): root = "C:\\owned" scopes = {root, root + "\\資料"} assert windows_watch.parse_notifications(_notify(), root, scopes).folders == (root + "\\資料",) - assert windows_watch.parse_notifications(_notify("new\\deep\\file"), root, scopes).folders == (root,) + # The window reads the deepest folder it scanned there, whose listing reads the new ones whole. + assert windows_watch.parse_notifications(_notify("new\\deep\\file"), root, scopes).folders == ( + root + "\\new\\deep",) assert windows_watch.parse_notifications(b"", root, scopes).full +def test_usn_feed_reads_on_from_a_handed_over_position_or_asks_for_the_whole_root(monkeypatch): + folder = change.WatchFolder("C:\\owned", 7, 22) + kernel = SimpleNamespace(CreateFileW=lambda *_args: 42, CloseHandle=lambda _handle: None) + reads, batches, positions = [], [], [] + cancel = threading.Event() + + def control(_kernel, _handle, code, data=b""): + if code == windows_watch._QUERY_USN: + return windows_watch._JOURNAL.pack(9, 50, 300, 50, 0, 4096, 1024) + reads.append(windows_watch._READ.unpack(data)[0]) + cancel.set() + return struct.pack(" native limit") - monkeypatch.setattr(following, "watch", failed) + monkeypatch.setattr(following, "watch_scope", failed) monkeypatch.setattr(following, "supported", lambda: True) window._actions["follow_changes"].setChecked(True) - _wait(qapp, lambda: bool(window._follow.worker.error)) + _wait(qapp, lambda: window._follow.worker is not None and bool(window._follow.worker.error)) calls = [] monkeypatch.setattr(window, "rescan_folder", calls.append) window._follow._merge(ChangeBatch(full=True)) @@ -211,16 +211,15 @@ def failed(*_args, **_kwargs): assert window._follow.worker is None -def test_unrelated_dirty_branches_survive_the_first_refresh(window, qapp, tmp_path, monkeypatch): +def test_unrelated_changed_folders_are_read_in_one_refresh(window, qapp, tmp_path, monkeypatch): source, _root = _source(window, tmp_path) controller = _enable(window, qapp, monkeypatch) calls = [] - monkeypatch.setattr(window, "rescan_folder", calls.append) + monkeypatch.setattr(window, "refresh_folders", calls.append) _dirty(controller, source / "資料", source / "另一個") controller.tick() - assert len(calls) == 1 and len(controller._pending.folders) == 1 - controller.tick() - assert {node.path for node in calls} == {str(source / "資料"), str(source / "另一個")} + assert len(calls) == 1 + assert {node.path for node, _whole in calls[0]} == {str(source / "資料"), str(source / "另一個")} assert not controller._pending.folders @@ -235,7 +234,8 @@ def test_partial_or_combined_scan_refuses_following_without_native_setup(window, def forbidden(*_args, **_kwargs): pytest.fail("partial scan started a native feed") - monkeypatch.setattr(following, "watch", forbidden) + monkeypatch.setattr(following, "watch_scope", forbidden) + monkeypatch.setattr(following, "captured_scope", forbidden) monkeypatch.setattr(following, "supported", lambda: True) window._actions["follow_changes"].setChecked(True) assert window._follow.worker is None and window._follow._status == "follow_incomplete" @@ -300,7 +300,7 @@ def test_native_event_replaces_one_branch_preserves_other_source_and_joins(windo from je_file_tree.core import windows_watch # Volume-wide USN changes can legitimately require a full rescan; this case proves branch notifications. - def unavailable(*_args): + def unavailable(*_args, **_kwargs): raise PermissionError("Native GUI branch test uses recursive directory notifications") monkeypatch.setattr(windows_watch, "_watch_usn", unavailable) @@ -309,6 +309,7 @@ def unavailable(*_args): file = source / "另一個" / "untouched" proof = file.stat().st_ino, hashlib.sha256(file.read_bytes()).hexdigest() old_branch = next(child for child in root.children if child.name == "資料") + nested = next(child for child in old_branch.children if child.name == "nested") assert any(item.node is old_branch for item in window.results.charts.tree.rows()) window._actions["follow_changes"].setChecked(True) _wait(qapp, lambda: window._follow.worker is not None and bool(window._follow.worker.backend)) @@ -321,11 +322,13 @@ def unavailable(*_args): _wait(qapp, lambda: root.size == 4 + 9 + 22528 and window._worker is None and not window._analysers) assert window.results.outcome.result.root is root and untouched in root.children assert (file.stat().st_ino, hashlib.sha256(file.read_bytes()).hexdigest()) == proof + # Only that folder was read again, in place: it and the subfolder it kept are the same nodes. current_branch = next(child for child in root.children if child.name == "資料") - assert current_branch is not old_branch + assert current_branch is old_branch and any(child.name == "新增.bin" for child in current_branch.children) + assert nested in current_branch.children and nested.parent is current_branch assert any(item.node is current_branch for item in window.results.charts.tree.rows()) - assert not any(item.node is old_branch for item in window.results.charts.tree.rows()) - assert cancelled.is_set() and finished.count() == 1 + # The feed kept reporting while the folder was read; no folder changed, so it simply carries on. + assert window._follow.worker is worker and not cancelled.is_set() and finished.count() == 0 active = window._follow.worker assert active is not None active_cancelled = active.cancel @@ -334,3 +337,52 @@ def unavailable(*_args): _wait(qapp, lambda: window._close_ready) assert active_finished.count() == 1 assert active_cancelled.is_set() + + +def test_a_change_directly_in_the_root_reads_only_its_own_entries(window, qapp, tmp_path, monkeypatch): + source, root = _source(window, tmp_path) + controller = _enable(window, qapp, monkeypatch) + scans = [] + monkeypatch.setattr(window, "start_scan", lambda path, **_kwargs: scans.append(path)) + kept = {child.name: child for child in root.children} + feed = controller.worker + (source / "另一個" / "untouched").write_bytes(b"not read again") # inside a kept subfolder + (source / "新檔案").write_bytes(b"n" * 7) + _dirty(controller, source) + controller.tick() + _wait(qapp, lambda: window._worker is None and not window._analysers and root.size == 4 + 9 + 7) + assert scans == [] and window.results.outcome.result.root is root + assert controller.worker is feed and not feed.cancel.is_set() and not controller._capture # no folder changed + assert all(next(child for child in root.children if child.name == name) is node for name, node in kept.items()) + assert next(child for child in kept["另一個"].children).size == 9 # what it holds was kept, not read + + +def test_the_feed_keeps_reporting_while_a_folder_is_read_and_hands_over_after(window, qapp, tmp_path, monkeypatch): + source, root = _source(window, tmp_path) + controller = _enable(window, qapp, monkeypatch) + before = controller.worker + (source / "資料" / "新增.bin").write_bytes(b"x" * 3) + (source / "資料" / "新資料夾").mkdir() # a new folder: the scope is captured again and a new feed takes over + _dirty(controller, source / "資料") + controller.tick() + assert window._worker is not None and before.isRunning() and not before.cancel.is_set() + before.record(ChangeBatch((str(source / "另一個"),))) # reported while the folder was being read + (source / "另一個" / "later").write_bytes(b"y" * 5) + _wait(qapp, lambda: controller.worker is not before and bool(controller.worker.backend)) + _wait(qapp, lambda: window._worker is None and not window._analysers) + controller.tick() + assert before.cancel.is_set() + controller._since -= 2 + controller.tick() + _wait(qapp, lambda: window._worker is None and not window._analysers and root.size == 4 + 3 + 9 + 5) + + +def test_a_folder_removed_before_it_is_read_reads_the_folder_that_held_it(window, qapp, tmp_path, monkeypatch): + source, root = _source(window, tmp_path) + controller = _enable(window, qapp, monkeypatch) + nested = next(child for child in root.children if child.name == "資料").children + nested = next(child for child in nested if child.name == "nested") + (source / "資料" / "nested").rmdir() + window.refresh_folders([(nested, False)]) + _wait(qapp, lambda: window._worker is None and not window._analysers) + assert controller._pending.folders == (str(source / "資料"),) diff --git a/test/test_folder_refresh.py b/test/test_folder_refresh.py new file mode 100644 index 00000000..45988a2b --- /dev/null +++ b/test/test_folder_refresh.py @@ -0,0 +1,85 @@ +"""Reading one changed folder's own entries again keeps its subfolders and corrects every total above it.""" + +import os +import shutil + +from je_file_tree.core.refresh import kept_subfolders, refill, refreshed_errors +from je_file_tree.core.scanner import KEPT, ScanOptions, scan + + +def _tree(tmp_path): + root = tmp_path / "來源" + (root / "a" / "deep").mkdir(parents=True) + (root / "b").mkdir() + (root / "a" / "deep" / "one").write_bytes(b"1" * 10) + (root / "b" / "two").write_bytes(b"2" * 20) + (root / "top").write_bytes(b"t" * 5) + return root, scan(root).root + + +def _child(folder, name): + return next(child for child in folder.children if child.name == name) + + +def test_a_listing_reads_only_the_folder_and_keeps_its_subfolders_in_place(tmp_path): + source, root = _tree(tmp_path) + a, deep = _child(root, "a"), _child(_child(root, "a"), "deep") + (source / "a" / "deep" / "one").write_bytes(b"1" * 99) # below a kept subfolder: not read now + (source / "top").write_bytes(b"t" * 7) + (source / "c").mkdir() + (source / "c" / "three").write_bytes(b"3" * 30) + keep = kept_subfolders(root) + assert {name for name, _device, _number in keep} == {"a", "b"} + + fresh = scan(source, options=ScanOptions(keep=keep)) + assert _child(fresh.root, "a").error == KEPT and not _child(fresh.root, "a").children + old_top = _child(root, "top") + assert refill(root, fresh.root) == frozenset({"a", "b"}) + + assert _child(root, "a") is a and a.parent is root and _child(a, "deep") is deep and deep.size == 10 + assert _child(root, "c").size == 30 and _child(_child(root, "c"), "three").parent is _child(root, "c") + assert _child(root, "top").size == 7 and old_top.parent is None + assert root.size == 10 + 20 + 7 + 30 and root.file_count == 4 and root.dir_count == 4 + assert [child.name for child in root.children] == ["c", "b", "a", "top"] + + +def test_refilling_a_subfolder_corrects_the_folders_above_it(tmp_path): + source, root = _tree(tmp_path) + b = _child(root, "b") + (source / "b" / "more").write_bytes(b"m" * 40) + (source / "b" / "two").unlink() + assert refill(b, scan(source / "b", options=ScanOptions(keep=kept_subfolders(b))).root) == frozenset() + assert b.size == 40 and [child.name for child in b.children] == ["more"] + assert root.size == 10 + 40 + 5 and root.file_count == 3 and _child(root, "b") is b + + +def test_a_subfolder_replaced_since_the_scan_is_read_again(tmp_path): + source, root = _tree(tmp_path) + keep = kept_subfolders(root) + old_b = _child(root, "b") + shutil.rmtree(source / "b") + (source / "b").mkdir() + (source / "b" / "new").write_bytes(b"n" * 3) + fresh = scan(source, options=ScanOptions(keep=keep)) + if _child(fresh.root, "b").error == KEPT: # the system gave the new folder the old one's file number + return + assert refill(root, fresh.root) == frozenset({"a"}) + assert _child(root, "b") is not old_b and _child(root, "b").size == 3 and old_b.parent is None + + +def test_nothing_changes_when_a_kept_subfolder_left_the_tree_meanwhile(tmp_path): + source, root = _tree(tmp_path) + fresh = scan(source, options=ScanOptions(keep=kept_subfolders(root))) + _child(root, "a").detach() + before = list(root.children), root.size + assert refill(root, fresh.root) is None + assert (list(root.children), root.size) == before + + +def test_errors_inside_kept_subfolders_stay_and_the_rest_are_replaced(tmp_path): + folder = str(tmp_path / "f") + errors = [(folder, "partial"), (os.path.join(folder, "kept", "x"), "denied"), + (os.path.join(folder, "gone"), "denied"), (str(tmp_path / "other"), "denied")] + fresh = [(os.path.join(folder, "new"), "denied")] + assert refreshed_errors(errors, folder, frozenset({"kept"}), fresh) == [ + (os.path.join(folder, "kept", "x"), "denied"), (str(tmp_path / "other"), "denied"), *fresh] diff --git a/test/test_mft_scan.py b/test/test_mft_scan.py index 3afcb23b..9a9e9a1e 100644 --- a/test/test_mft_scan.py +++ b/test/test_mft_scan.py @@ -340,7 +340,7 @@ def __init__(self, path, options, *arguments, **named): super().__init__(path, options, *arguments, **named) monkeypatch.setattr(main, "ScanWorker", Recording) - monkeypatch.setattr(main, "queue_worker", lambda *_arguments: None) # built and owned, never started + monkeypatch.setattr(main, "queue_worker", lambda *_arguments, **_options: None) # built and owned, never started inner = window.results.tree_model.root.children[0] assert window._scan_options().mft window.rescan_folder(inner) diff --git a/test/test_shell_dialog_lifecycle.py b/test/test_shell_dialog_lifecycle.py index 01486051..9dc70c87 100644 --- a/test/test_shell_dialog_lifecycle.py +++ b/test/test_shell_dialog_lifecycle.py @@ -188,12 +188,12 @@ def test_opening_shell_settings_keeps_an_existing_live_reader(window, qapp, tmp_ window.results.show_outcome(scan_worker.analyse(scan(tmp_path))) entered, release = threading.Event(), threading.Event() - def watch(_root, _changed, _cancel, *, ready): + def watch(_scope, _changed, _cancel, *, ready, **_feed): ready("directory_changes") entered.set() release.wait(5) - monkeypatch.setattr(change_watch, "watch", watch) + monkeypatch.setattr(change_watch, "watch_scope", watch) monkeypatch.setattr(change_watch, "supported", lambda: True) monkeypatch.setattr(shell, "supported", lambda: True) monkeypatch.setattr(shell, "installed", lambda: False) diff --git a/test/test_worker_lifecycle.py b/test/test_worker_lifecycle.py index b60b08cb..37ea3ccb 100644 --- a/test/test_worker_lifecycle.py +++ b/test/test_worker_lifecycle.py @@ -198,13 +198,13 @@ def checking(path): def _held_follow(window, qapp, monkeypatch): release, entered = threading.Event(), threading.Event() - def watch(_root, _changed, cancel, *, ready): + def watch(_scope, _changed, cancel, *, ready, **_feed): ready("directory_changes") entered.set() cancel.wait(5) release.wait(5) - monkeypatch.setattr(change_watch, "watch", watch) + monkeypatch.setattr(change_watch, "watch_scope", watch) monkeypatch.setattr(change_watch, "supported", lambda: True) window._actions["follow_changes"].setChecked(True) _pump(qapp, entered.is_set) @@ -220,13 +220,13 @@ def test_follow_reconfiguration_queues_only_latest_reader_without_gui_wait(windo release = threading.Event() calls = [] - def watch(_root, _changed, cancel, *, ready): + def watch(_scope, _changed, cancel, *, ready, **_feed): calls.append(QThread.currentThread()) ready("directory_changes") cancel.wait(5) release.wait(5) - monkeypatch.setattr(change_watch, "watch", watch) + monkeypatch.setattr(change_watch, "watch_scope", watch) monkeypatch.setattr(change_watch, "supported", lambda: True) action = window._actions["follow_changes"] try: @@ -235,12 +235,12 @@ def watch(_root, _changed, cancel, *, ready): old = window._follow.worker for enabled in (False, True, False, True): action.setChecked(enabled) - newest = window._follow.worker assert old.cancel.is_set() and old.isRunning() - assert newest is not old and calls == [old] + assert window._follow.worker is not old and calls == [old] QTimer.singleShot(20, release.set) - _pump(qapp, lambda: len(calls) == 2 and bool(newest.backend)) - assert release.is_set() and calls == [old, newest] + _pump(qapp, lambda: len(calls) == 2 and bool(window._follow.worker and window._follow.worker.backend)) + newest = window._follow.worker + assert release.is_set() and newest is not old and calls == [old, newest] assert kept.read_bytes() == b"kept" finally: release.set() @@ -289,20 +289,34 @@ def test_cancel_queued_branch_keeps_previous_result_and_clears_busy_state(window window.results.show_outcome(scan_worker.analyse(scan(branch.parent))) previous = window.results.outcome release, reader = _held_follow(window, qapp, monkeypatch) + held = _Held(window, release) # another of the window's threads, which a folder rescan waits for + held.start() calls = [] scanning = scan_worker.scan monkeypatch.setattr(scan_worker, "scan", lambda *args, **kwargs: calls.append(args) or scanning(*args, **kwargs)) try: window.rescan_folder(previous.result.root.children[0]) - assert reader.cancel.is_set() and reader.isRunning() + # The feed reads nothing of the tree: it reports on while the folder is read. + assert not reader.cancel.is_set() and reader.isRunning() assert window._worker is not None and not calls window.stop_scan() QTimer.singleShot(20, release.set) _pump(qapp, lambda: window._worker is None) assert release.is_set() and not calls assert window.results.outcome is previous and kept.read_bytes() == b"kept" + assert window._follow.worker is reader and not reader.cancel.is_set() finally: release.set() + held.wait() + + +class _Held(QThread): + def __init__(self, parent, release): + super().__init__(parent) + self.release = release + + def run(self): + self.release.wait(5) def test_replacing_a_scan_retains_only_latest_request_without_gui_wait(window, qapp, tmp_path, monkeypatch): diff --git a/test/test_workspace_gui.py b/test/test_workspace_gui.py index 8374e55b..ea1559dc 100644 --- a/test/test_workspace_gui.py +++ b/test/test_workspace_gui.py @@ -220,7 +220,7 @@ def test_a_delete_in_one_tab_offers_only_its_folder_in_another(workspace, qapp, second.results.show_outcome(scan_worker.analyse(scan(source))) peer_root = second.results.outcome.result.root if following: - monkeypatch.setattr(change_watch, "watch", lambda _root, _changed, cancel, *, ready: ( + monkeypatch.setattr(change_watch, "watch_scope", lambda _scope, _changed, cancel, *, ready, **_feed: ( ready("directory_changes"), cancel.wait())) monkeypatch.setattr(change_watch, "supported", lambda: True) second._actions["follow_changes"].setChecked(True) From 6e4710b4234eeb22687947a6283c4e573a9db4e4 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 20 Nov 2026 05:09:02 +0800 Subject: [PATCH 2/4] Draw each page icon once per colour, and skip a needless folder re-read Verified Follow changes natively on the USN journal and on directory notifications: a change in the scanned folder reads only that folder, a folder added during the hand-over is caught from the held journal position, and nothing rescans the whole root. Two costs found on the way: the result navigation redrew every page icon whenever its texts were updated (about 100 ms after each refresh), and a folder removed before it was read asked for its folder again even when that folder had just been read in the same pass. --- docs/updates/2026-11.md | 30 ++++++++++++++++++++++++++++++ docs/updates/README.md | 1 + je_file_tree/gui/fluent_icons.py | 27 ++++++++++++++++++++------- je_file_tree/gui/folder_refresh.py | 3 ++- progress.md | 2 +- test/test_change_watch_gui.py | 11 +++++++++++ test/test_fluent_icons.py | 7 +++++++ 7 files changed, 72 insertions(+), 9 deletions(-) diff --git a/docs/updates/2026-11.md b/docs/updates/2026-11.md index c5e0b8b3..c7c3d7bd 100644 --- a/docs/updates/2026-11.md +++ b/docs/updates/2026-11.md @@ -732,3 +732,33 @@ - **未驗證**:USN 位置交接只用測試替身驗證(本機需要系統管理員才能讀取變更日誌);Linux inotify 的新舊監看並存(監看數量上限)只由 CI 的原生測試涵蓋;沒有在使用者的電腦上操作。 - **檔案**:`je_file_tree/core/refresh.py`(新)、`core/scanner.py`、`core/change_watch.py`、`core/windows_watch.py`;`je_file_tree/gui/folder_refresh.py`(新)、`gui/change_watch.py`、`gui/main_window.py`、`gui/results_view.py`、`gui/tree_model.py`、`gui/worker_lifecycle.py`、`gui/strings.py`;`architecture.md`、`docs/core-api.md`、`docs/guide.md` 與四個翻譯;`progress.md`;測試檔案。 - **待辦**:原生環境驗證記錄於 `progress.md` #144。 + +## U-20261120-02 · 2026-11-20 · 在本機原生驗證「跟隨變更」只讀變動資料夾,並修正兩處多餘工作 · #validation #perf #gui + +- **原因**:使用者要求驗證 U-20261120-01。本機以系統管理員執行,C: 的 NTFS 變更日誌可以讀取,因此 USN 位置交接可以在原生環境驗證,不只用測試替身。 +- **驗證方式**:在 scratchpad 中自建的測試資料夾上,以真實的原生監看與隱藏視窗(`WA_DontShowOnScreen`)、真正的 `QEventLoop`,並和 `app.main` 一樣以 `pace_workers()` 接上讓路閘門。只刪除測試自己建立的資料夾。 +- **結果(USN,原生)**: + - 監看閒置 30 秒:整個磁碟區沒有任何回報,也沒有要求重掃整個根目錄。 + - 停止監看後在子資料夾新增檔案,再從保留的位置啟動新監看:這個變動被回報。 +- **結果(視窗,USN 與目錄通知各一次)**: + + | 情況 | USN | 目錄通知 | + |---|---|---| + | 根目錄新增一個檔案 | 3.07 s;只列出根目錄,保留 2 個子資料夾;根節點、子資料夾節點與監看都不變 | 2.54 s;同左 | + | 新增資料夾,並在交接期間於其中再新增檔案 | 5.00 s;兩個檔案都出現;新監看從保留位置接著讀 | 5.03 s;兩個檔案都出現 | + | 刪除子資料夾 | 3.00 s;結果正確 | 2.00 s;結果正確 | + | 閒置 60 秒 | 沒有整個重掃 | 沒有整個重掃 | + + 所有情況都沒有重掃整個根目錄。 +- **發現並修正**: + - 結果導覽列每次更新文字都重畫每個分頁的圖示(每頁 12 張圖),約 100 ms,「跟隨變更」每次更新後都會發生。`fluent_icons.icon` 改為每種顏色組合只畫一次。量測:`refill_folders` 由 205 ms 降為 44–48 ms。 + - 資料夾在讀取前被刪除,而包含它的資料夾在同一批也讀過時,不再要求重讀一次包含它的資料夾。 +- **卡頓量測**(20,000 個資料夾、100,000 個檔案,共 120,208 個項目,每 5 ms 計時一次): + - 接上讓路閘門時(與實際程式相同),五次量測中最長的單次卡頓:根目錄新增檔案 29–53 ms,根目錄新增資料夾(會重新讀取監看範圍)34–85 ms。 + - 未接上閘門時為 168–804 ms;最初的量測就是漏接了閘門,不代表實際程式。 + - 重新讀取監看範圍本身 0.43 s,持有直譯器鎖的時間都不超過計時精度(16 ms)。 +- **量測工具造成的當機**:量測腳本中從另一個執行緒讀取執行中執行緒堆疊的取樣器,在 Python 3.14 上會引發存取違規(3 次中 2 次);拿掉取樣器後 4 次都沒有當機。這是量測工具本身的問題,與 FileTree 無關。 +- **測試**:`test_fluent_icons.py` 新增 1 個、`test_change_watch_gui.py` 新增 1 個;`test_fluent_icons.py`、`test_change_watch_gui.py`、`test_folder_refresh.py` 共 41 個測試通過;`ruff check .` 通過。 +- **未驗證**:Linux inotify 新舊監看並存與監看數量上限;使用者實際資料夾上的長時間使用。 +- **檔案**:`je_file_tree/gui/fluent_icons.py`、`gui/folder_refresh.py`;`test/test_fluent_icons.py`、`test/test_change_watch_gui.py`;`progress.md`。 +- **待辦**:`progress.md` #144(已移除 USN 部分)。 diff --git a/docs/updates/README.md b/docs/updates/README.md index c2b8d9c7..02f48ff2 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -70,6 +70,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261120-02 | 2026-11-20 | 在本機原生驗證「跟隨變更」只讀變動資料夾,並修正兩處多餘工作 | #validation #perf #gui | [2026-11](2026-11.md) | | U-20261120-01 | 2026-11-20 | 「跟隨變更」只重新讀取變動的資料夾,不再重掃整個掃描 | #perf #gui #scan | [2026-11](2026-11.md) | | U-20261119-07 | 2026-11-19 | 大型資料夾的加總、排序與清理比對也分頁讓路給視窗 | #perf #fix | [2026-11](2026-11.md) | | U-20261119-06 | 2026-11-19 | 只拖動捲軸也會卡頓:清理建議背景工作沒有讓路給視窗 | #perf #gui #fix | [2026-11](2026-11.md) | diff --git a/je_file_tree/gui/fluent_icons.py b/je_file_tree/gui/fluent_icons.py index f4a59f3c..0a6b3fdf 100644 --- a/je_file_tree/gui/fluent_icons.py +++ b/je_file_tree/gui/fluent_icons.py @@ -93,14 +93,27 @@ def _question() -> QPainterPath: NAMES = tuple(_SHAPES) +_DRAWN: dict[tuple[str, int, int], QIcon] = {} # one per icon and palette; a theme change adds a few more + + def icon(name: str, colour: QColor, disabled: QColor | None = None) -> QIcon: - """The named line icon in ``colour`` (and ``disabled`` for the disabled state).""" - strokes = _SHAPES[name]() - result = QIcon() - for mode, ink in ((QIcon.Mode.Normal, colour), (QIcon.Mode.Disabled, disabled or colour)): - for size in _SIZES: - result.addPixmap(_render(strokes, ink, size), mode) - return result + """The named line icon in ``colour`` (and ``disabled`` for the disabled state). + + Drawn once per colour pair: the result navigation asks for every page's icon each time its texts + are updated, and drawing them all took about 100 ms, after every Follow changes refresh too + (U-20261120-02). + """ + disabled = disabled or colour + key = (name, colour.rgba(), disabled.rgba()) + drawn = _DRAWN.get(key) + if drawn is None: + strokes = _SHAPES[name]() + drawn = QIcon() + for mode, ink in ((QIcon.Mode.Normal, colour), (QIcon.Mode.Disabled, disabled)): + for size in _SIZES: + drawn.addPixmap(_render(strokes, ink, size), mode) + _DRAWN[key] = drawn + return drawn def palette_icon(name: str, palette: QPalette) -> QIcon: diff --git a/je_file_tree/gui/folder_refresh.py b/je_file_tree/gui/folder_refresh.py index 001d4fd5..9b78036b 100644 --- a/je_file_tree/gui/folder_refresh.py +++ b/je_file_tree/gui/folder_refresh.py @@ -105,8 +105,9 @@ def _refreshed(window: MainWindow, chosen: list[tuple[Node, bool]], results: tup window._scan_ended() listings = [(node, result) for (node, _whole), result in zip(chosen, results, strict=True) if result is not None] follow = window._follow + read = {id(node) for node, _result in listings} for (node, _whole), result in zip(chosen, results, strict=True): - if result is None: + if result is None and (node.parent is None or id(node.parent) not in read): # its folder shows it gone follow.request(node.parent.path if node.parent is not None else None) failed = window.results.refill_folders(listings) for node in failed: diff --git a/progress.md b/progress.md index cd8559d3..4a8db8d8 100644 --- a/progress.md +++ b/progress.md @@ -23,7 +23,7 @@ ### 掃描 - **#142**(第二優先)[待驗證] 由主檔案表讀取 NTFS 磁碟的掃描(U-20261115-05、-07)只在一台機器的一顆系統磁碟上量測與比對。尚未驗證:檔案紀錄為 4096 位元組的磁碟(4Kn);檔案表本身高度分散的磁碟;資料磁碟與外接磁碟;啟用系統壓縮(CompactOS)或含大量雲端預留位置的磁碟;真正的 Windows 10;冷快取下與列出資料夾的秒數比較;CI 的 Windows 工作能否讀取檔案表(讀不到時 `test_mft_scan.py` 的實機測試會略過);已發行的 wheel 與獨立版是否含新版模組;有日光節約時間的時區下原生時間文字與 Python 是否相同(U-20261115-10)。另有 85 個資料夾的大小與 Windows 回報不同,原因未查明;這些項目在操作前會被判定為已變更而拒絕,重新掃描該資料夾即可。 -- **#144**(第二優先)[待驗證] 「跟隨變更」只重新讀取變動資料夾(U-20261120-01),目前只以 Windows 目錄通知(非系統管理員)與測試替身驗證。尚未驗證:以系統管理員執行時,NTFS USN 監看能否從保留的位置接著讀,交接期間不漏掉變動;Linux inotify 新舊監看短暫並存時,是否會超過監看數量上限;在使用者實際使用、變動頻繁的資料夾上,重新讀取監看範圍的背景負擔。 +- **#144**(第二優先)[待驗證] 「跟隨變更」只重新讀取變動資料夾(U-20261120-01),Windows 的 NTFS USN 與目錄通知已在本機原生驗證(U-20261120-02)。尚未驗證:Linux inotify 新舊監看短暫並存時,是否會超過監看數量上限;在使用者實際使用、變動頻繁的資料夾上,重新讀取監看範圍的背景負擔。 - **#138**(第三優先)[待驗證] 選用的原生掃描模組 `je_file_tree._native` 已隨 0.2.9 發行(PyPI 的 Windows wheel 與 Windows 獨立版),發行本身已驗證。尚未驗證的環境:真正的 Windows 10;未安裝 Visual C++ 可轉散發套件的乾淨機器(模組須可載入並被採用);非 NTFS 磁碟區中的 exFAT 與 ReFS(FAT32 已驗證),須確認掃描起點比對會正確決定是否使用原生讀取;ARM64 的 Windows(目前只發行 x64 wheel)。目前沒有這些環境。任何一項失敗時,程式仍以 Python 讀取掃描,結果相同。 - **#35**(第二優先)[受阻] 須在實際緩慢的遠端 SMB 連線上,量測 UNC 路徑與對應磁碟機在不同背景工作數量下的處理量,再建議該環境的並行程度。使用者目前未提供遠端共用資料夾或慢速連線驗證環境,本機回送連線的原生量測無法證明遠端效能。 - **#75**(第一優先)[受阻] 須透過原生目錄走訪防護,驗證 macOS 與其他 POSIX 系統上同一裝置內的掛載邊界。使用者目前未提供可拋棄的對應驗證環境。FreeBSD 的 nullfs 具有不同的 `st_dev`,無法證明此情境。未掛載的 APFS 同容器磁碟區容量仍未知。圖形介面與命令列的容量帳目仍為估計值;無法獨立量測的共用空間、保留空間、檔案系統中繼資料、遺漏位元組及其他磁碟區總量,均須維持未知。直接顯示垃圾桶標籤與 Finder 清空功能仍由 #21 追蹤。 diff --git a/test/test_change_watch_gui.py b/test/test_change_watch_gui.py index aba20bc5..9b4cf350 100644 --- a/test/test_change_watch_gui.py +++ b/test/test_change_watch_gui.py @@ -386,3 +386,14 @@ def test_a_folder_removed_before_it_is_read_reads_the_folder_that_held_it(window window.refresh_folders([(nested, False)]) _wait(qapp, lambda: window._worker is None and not window._analysers) assert controller._pending.folders == (str(source / "資料"),) + + +def test_a_removed_folder_whose_folder_was_read_too_asks_for_nothing_more(window, qapp, tmp_path, monkeypatch): + source, root = _source(window, tmp_path) + controller = _enable(window, qapp, monkeypatch) + branch = next(child for child in root.children if child.name == "資料") + nested = next(child for child in branch.children if child.name == "nested") + (source / "資料" / "nested").rmdir() + window.refresh_folders([(branch, False), (nested, False)]) + _wait(qapp, lambda: window._worker is None and not window._analysers) + assert nested not in branch.children and not controller._pending.folders diff --git a/test/test_fluent_icons.py b/test/test_fluent_icons.py index efb73b93..d94c0eef 100644 --- a/test/test_fluent_icons.py +++ b/test/test_fluent_icons.py @@ -50,3 +50,10 @@ def test_sidebar_and_chooser_icons_follow_the_theme(window, qapp, sample_tree): def test_tool_bar_actions_show_icons_beside_their_text(window): for key in ("open", "rescan", "stop", "help"): assert not window._actions[key].icon().isNull(), key + + +def test_an_icon_is_drawn_once_per_colour_pair(qapp): + first = icon(NAMES[0], QColor("#102030"), QColor("#405060")) + assert icon(NAMES[0], QColor("#102030"), QColor("#405060")).cacheKey() == first.cacheKey() + other = icon(NAMES[0], QColor("#a0b0c0"), QColor("#405060")) + assert other.cacheKey() != first.cacheKey() and _drawn_in(other, "#a0b0c0") From a11248970acf728ec28975e55818943d1af178b9 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 20 Nov 2026 15:43:57 +0800 Subject: [PATCH 3/4] Follow changes keeps watching through operations, and follows on macOS Source operations now stop only the capture, which reads the tree; the feed keeps reporting, so an operation leaves no gap and a cancelled review no longer reads the whole root. A feed lost during a Recycle Bin move, permanent delete or Undo is still withdrawn by it. A hard link made anywhere on an NTFS volume no longer reads the whole scan: the file's names are looked up by file number and only folders holding one inside the scan are read. The journal is read on at once while records wait, directory notifications get a 256 KiB buffer, and an inotify feed reads the folders it newly watches once armed. macOS follows through one FSEvents stream, which reports folders and says when it dropped events or could not keep up. --- architecture.md | 29 ++++-- docs/core-api.md | 12 ++- docs/guide.md | 2 +- docs/guide_ja.md | 2 +- docs/guide_ko.md | 2 +- docs/guide_zh-CN.md | 2 +- docs/guide_zh-TW.md | 2 +- docs/updates/2026-11.md | 21 ++++ docs/updates/README.md | 1 + je_file_tree/core/change_watch.py | 6 +- je_file_tree/core/mac_watch.py | 140 ++++++++++++++++++++++++++ je_file_tree/core/windows_watch.py | 151 ++++++++++++++++++++++++++--- je_file_tree/gui/change_watch.py | 92 ++++++++++++------ je_file_tree/gui/strings.py | 34 ++++--- progress.md | 2 +- test/test_change_watch.py | 113 ++++++++++++++++++++- test/test_change_watch_gui.py | 74 +++++++++++--- test/test_worker_lifecycle.py | 18 ++-- 18 files changed, 602 insertions(+), 101 deletions(-) create mode 100644 je_file_tree/core/mac_watch.py diff --git a/architecture.md b/architecture.md index a5ebdc69..17c967e7 100644 --- a/architecture.md +++ b/architecture.md @@ -322,7 +322,10 @@ copy/link/CJK/source-preservation checks, with complete owned-fixture cleanup. folders and 65,536 shared inodes. `windows_watch` reads only existing NTFS USN journals and uses one recursive overlapped directory handle when unavailable; partial shared-file roots refuse that fallback because external aliases are not covered. `linux_watch` uses one inotify descriptor with -anchored folder and captured shared-inode watches, exposing kernel limit failures. Bounded events +anchored folder and captured shared-inode watches, exposing kernel limit failures. `mac_watch` +runs one FSEvents stream on the scanned folder (U-20261120-03): it reports folders, says when it +dropped events (whole root) or could not keep up (that folder, read whole), and, like the directory +handle, refuses partial roots with shared files because outside names are not reported. Bounded events map to captured folder paths; overflow/gaps request full scans, malformed records fail visibly. Native cancellation joins outstanding Windows I/O before freeing storage. Notifications are observational rescan requests and never operation authority. Reviewed native CI confirms ordinary @@ -335,7 +338,7 @@ signal queues stay bounded. The feed reads nothing of the tree and has no Qt par read again while it keeps reporting (`queue_worker(readers=...)` lets a scan wait only for the readers it stopped itself). A report names the folder whose own entries changed (`ChangeBatch.folders`), or, past `MAX_CHANGED`, the folder holding many changes, read whole (`ChangeBatch.trees`). The -one-second timer defers during scans/analysis, source reviews, modal questions and Undo offers, maps +quarter-second timer (changes coalesced for half a second) defers during scans/analysis, source reviews, modal questions and Undo offers, maps each path to the deepest scanned folder (`core.node.folder_holding`) and hands them all to `gui.folder_refresh`: one `ListingWorker` lists each folder with `ScanOptions.keep` (its read subfolders, by name and identity, stay unread) and `core.refresh.refill` puts the listing into the @@ -343,14 +346,20 @@ same `Node` with the kept subfolders hung back, so a change in the scan root rea own entries and the view keeps its place. A kept subfolder replaced meanwhile is read whole next; a folder removed meanwhile makes its parent's listing be read. When a listing found, lost or replaced a subfolder, or listed a file with another name, the scope is captured again (otherwise the running feed -still covers it all and carries on) and the new feed takes over only once armed; an NTFS journal feed starts from the position held when -the refresh began (or when a source operation stopped the feed), so nothing between is missed. Only -lost or overflowing feeds and an operation gap without a journal position read the whole root; there is -no periodic full reconciliation. A finished Trash move, permanent delete or Undo withdraws its own gap request -(`FollowChanges.resume`), in its tab and, through `MainWindow._tell_peers`, in every other tab of the -OperationGroup, which offers only its own folders holding the changed paths. New roots discard old data. -Shared OperationGroup exclusion quiesces peer monitors before source mutation/tree changes. Native -failures remain visible without retry loops; opt-out/tab close cancel/join through wait_for. +still covers it all and carries on) and the new feed takes over only once armed; an NTFS journal feed +starts from the position held when the refresh began, so nothing between is missed, and an inotify +feed has the folders it watches newly read again once armed. Source operations (and their peers in the +OperationGroup) stop only the capture, which reads the tree (`FollowChanges.quiesce`, U-20261120-03): +the feed holds no handle that keeps an entry from moving, so it keeps reporting and the operation leaves +no gap; the scope is captured again once the operation is over. A feed lost during an operation counts +as its gap, which a finished Trash move, permanent delete or Undo withdraws (`FollowChanges.resume`), in +its tab and, through `MainWindow._tell_peers`, in every other tab of the OperationGroup, which offers only +its own folders holding the changed paths. A journal feed reads on at once while records wait, and a +file given or stripped of a name anywhere on the volume is looked up by file number (`OpenFileById`, +`FindFirstFileNameW`): only folders holding one of its names inside the scan are read. Directory +notifications use a 256 KiB buffer (64 KiB over a network). Only lost or overflowing feeds read the +whole root; there is no periodic full reconciliation. New roots discard old data. Native failures +remain visible without retry loops; opt-out/tab close cancel/join through wait_for. Scans start only on a user request (U-20261114-18): opening/dropping a folder, Rescan, a branch rescan, or the FollowChanges opt-in above. Source operations never start one. Approved Trash and diff --git a/docs/core-api.md b/docs/core-api.md index 44afa63a..2b9641b3 100644 --- a/docs/core-api.md +++ b/docs/core-api.md @@ -700,7 +700,10 @@ Windows reads an existing NTFS USN journal without creating/deleting one; unavai fall back to one recursive ReadDirectoryChangesW handle. Partial roots with captured shared files require readable USN, because recursive directory notifications cannot cover outside aliases. Linux uses one inotify descriptor, anchored folder watches and captured shared-inode watches; -kernel resource limits are visible failures. Unsupported/corrupt records raise OSError/ValueError. +kernel resource limits are visible failures. macOS uses one FSEvents stream on the scanned +folder (`mac_watch.parse_fsevents` maps its directory events, `MustScanSubDirs` to `trees`, dropped or +wrapped events and root/mount changes to a full scan); partial roots with shared files are refused. +Unsupported/corrupt records raise OSError/ValueError. Cancellation closes native resources; Windows cancels and joins outstanding kernel I/O before releasing its buffers. Setup/rearm races remain observational, and notifications confer no source operation authority. This core API remains independent of the GUI. @@ -722,9 +725,10 @@ Undo offers, then reads every reported folder's own entries again in one worker `core.refresh.refill(node, result.root)`). `ChangeBatch.trees` (too many changes below one folder) are read whole. The feed keeps reporting during that read; when the read found, lost or replaced a subfolder (or listed a hard-linked file), the next feed, armed on a fresh scope, replaces it only once -ready, a journal feed reading on `since=` the held `Position`. Lost events and -source-operation gaps without a journal position require a full-root scan; nothing reads the root -on a timer. New roots discard old events; opt-out/tab close cancel and join through wait_for. +ready, a journal feed reading on `since=` the held `Position`. Source operations stop only the +capture; the feed keeps reporting through them. `windows_watch.parse_usn(..., linked=set())` collects +files whose names changed so `_linked_folders` can look up their names by file number instead of +reading the whole root. Only lost events require a full-root scan; nothing reads the root on a timer. New roots discard old events; opt-out/tab close cancel and join through wait_for. Unsupported/incomplete scopes and native errors stay visible in plain-text status, with no automatic error retry loop. Notifications never grant mutation approval. diff --git a/docs/guide.md b/docs/guide.md index 5e08777d..7c840a90 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -111,7 +111,7 @@ it, and shows you the biggest folders and files first — in a folder tree, a co of the largest files. When you find something you no longer need, move it to the Recycle Bin right from the window. -In **Options → Follow changes** (off by default), each result tab follows its complete physical Windows/Linux scan independently. A background native USN/directory-notification/inotify feed coalesces bounded changes for a second; then FileTree reads again only the folders whose own entries changed, keeping the subfolders they already hold, so a file saved directly in the scanned folder does not rescan the whole scan. Several changed folders are read in one go; a folder with very many changes below it is read whole. The view keeps its place, expanded folders and selection. Scans, analysis, source-operation reviews, modal questions and live Undo offers defer refresh. The feed keeps watching while folders are read and hands over to a fresh one afterwards; an NTFS journal feed reads on from where it was, so nothing in between is missed. Only lost or overflowing events, and a source-operation gap without a journal to read on from, read the whole root again; nothing does so on a timer. A Recycle Bin move, a permanent delete and an Undo, in this tab or another, leave the results directly, and only the folders left on offer are read again. With hard links counted, the whole tree is recounted in memory after a refresh. Incomplete or combined roots and unsupported scopes cannot start following; native failures remain visible without automatic retry. Partial Windows roots with shared files require readable USN to cover outside aliases. Opt-out and tab close cancel and join the monitor. Notifications grant no source-operation authority. The independent Python API is `change_watch.watch` (or `captured_scope` and `watch_scope`); native CI verified Windows USN and Linux inotify, including Windows/Linux outside-alias writes. +In **Options → Follow changes** (off by default), each result tab follows its complete physical Windows, Linux or macOS scan independently. A background native USN/directory-notification/inotify/FSEvents feed coalesces bounded changes for half a second; then FileTree reads again only the folders whose own entries changed, keeping the subfolders they already hold, so a file saved directly in the scanned folder does not rescan the whole scan. Several changed folders are read in one go; a folder with very many changes below it is read whole. The view keeps its place, expanded folders and selection. Scans, analysis, source-operation reviews, modal questions and live Undo offers defer refresh. The feed keeps watching while folders are read, and during moves, deletes and other operations, and hands over to a fresh one afterwards; an NTFS journal feed reads on from where it was, so nothing in between is missed. A hard link made anywhere on the drive reads only the folders holding one of its names in the scan. Only lost or overflowing events read the whole root again; nothing does so on a timer. A Recycle Bin move, a permanent delete and an Undo, in this tab or another, leave the results directly (a feed overflowing meanwhile does not read the whole root), and only the folders left on offer are read again. With hard links counted, the whole tree is recounted in memory after a refresh. Incomplete or combined roots and unsupported scopes cannot start following; native failures remain visible without automatic retry. Partial Windows roots with shared files require readable USN to cover outside aliases. Opt-out and tab close cancel and join the monitor. Notifications grant no source-operation authority. The independent Python API is `change_watch.watch` (or `captured_scope` and `watch_scope`); native CI verified Windows USN and Linux inotify, including Windows/Linux outside-alias writes. **Options → Background monitor…** is off by default. Enable it to keep FileTree in the system tray when closing the window; **Quit** always joins all work and exits. If a tray is unavailable the window stays visible; losing a tray restores a hidden window. Once a minute, a worker reads OS available capacity and reports below-threshold crossings (10% by default) through supported system notifications and status/tooltips. OS preferences may suppress notifications; unknown capacity stays unknown. Choose up to 32 folders and a 1–720-hour interval for one-thread gentle scans saved to enabled local history. Scans leave result tabs and sources untouched; incomplete coverage stays explicit. Durable claims across processes prevent duplicate attempts; failed/canceled periods stay throttled and missed periods coalesce once. Foreground scans/source reviews cancel and join gentle work. Disabling joins workers and restores the window. `py -3 start_file_tree.py --background` skips elevation and hides only with already-enabled monitoring and an available tray; the flag never enables it. No automatic clean-up is performed. Core APIs `background.MonitorConfig`, `claim_scan`, `due_roots` and `SpaceWarnings` remain independent. diff --git a/docs/guide_ja.md b/docs/guide_ja.md index 9bf985db..a3346a4d 100644 --- a/docs/guide_ja.md +++ b/docs/guide_ja.md @@ -76,7 +76,7 @@ FileTree がスキャンするのは、ユーザーが求めたときだけで **ディスク容量がどこにあるかを確認します。** FileTree は、フォルダーまたはドライブ全体をスキャンし、その中のすべてのファイルを合計し、フォルダー ツリー、カラフルなツリーマップ、および最大ファイルのリストで、最大のフォルダーとファイルを最初に表示します。不要になったものを見つけたら、ウィンドウから直接ごみ箱に移動します。 -**オプション → 変更を追跡** (デフォルトではオフ) では、各結果タブは完全な物理 Windows/Linux スキャンを個別に追跡します。バックグラウンドのネイティブ USN/directory-notification/inotify フィードは制限された変更を 1 秒間まとめ、その後、自身の項目が変わったフォルダーだけを読み直し、すでに持っているサブフォルダーはそのまま残します。そのため、スキャンしたフォルダーに直接保存されたファイルでスキャン全体が読み直されることはありません。変更された複数のフォルダーは一度に読み直し、下で非常に多くの変更があったフォルダーは丸ごと読み直します。表示位置、展開したフォルダー、選択はそのまま残ります。スキャン、分析、元の項目に対する操作の確認、モーダル質問、およびライブ Undo オファーは更新を延期します。フォルダーを読み直す間も監視は続き、その後新しい監視に引き継ぎます。NTFS の変更ジャーナルは元の位置から読み続けるので、その間の変更も漏れません。ルート全体を読み直すのは、イベントの消失やあふれ、および続きを読めるジャーナルがないソース操作のギャップだけで、定期的に全体を読み直すことはありません。ごみ箱への移動、完全な削除、元に戻す操作は (このタブでも他のタブでも) 項目を結果から直接取り除き、読み直すのは通知に残ったフォルダーだけです。ハードリンクを数える場合は、更新後にツリー全体をメモリ上で数え直します。不完全または結合されたルートとサポートされていないスコープでは、フォローを開始できません。ネイティブ障害は自動再試行なしで表示されたままになります。共有ファイルを含む部分的な Windows ルートには、外部のエイリアスをカバーするための読み取り可能な USN が必要です。オプトアウトしてタブを閉じてキャンセルし、監視処理の終了を待機します。通知にはソース操作権限は付与されません。独立した Python API は `change_watch.watch` (または `captured_scope` と `watch_scope`) です。ネイティブ CI は、Windows USN および Linux inotify を検証しました (Windows/Linux 外部エイリアス書き込みを含む)。 +**オプション → 変更を追跡** (デフォルトではオフ) では、各結果タブは完全な物理 Windows、Linux、macOS スキャンを個別に追跡します。バックグラウンドのネイティブ USN/directory-notification/inotify/FSEvents フィードは制限された変更を 0.5 秒間まとめ、その後、自身の項目が変わったフォルダーだけを読み直し、すでに持っているサブフォルダーはそのまま残します。そのため、スキャンしたフォルダーに直接保存されたファイルでスキャン全体が読み直されることはありません。変更された複数のフォルダーは一度に読み直し、下で非常に多くの変更があったフォルダーは丸ごと読み直します。表示位置、展開したフォルダー、選択はそのまま残ります。スキャン、分析、元の項目に対する操作の確認、モーダル質問、およびライブ Undo オファーは更新を延期します。フォルダーを読み直す間も、移動や削除などの操作中も監視は続き、その後新しい監視に引き継ぎます。NTFS の変更ジャーナルは元の位置から読み続けるので、その間の変更も漏れません。ドライブのどこかで作られたハードリンクは、スキャン内にその名前があるフォルダーだけを読み直します。ルート全体を読み直すのはイベントの消失やあふれのときだけで、定期的に全体を読み直すことはありません。ごみ箱への移動、完全な削除、元に戻す操作は (このタブでも他のタブでも) 項目を結果から直接取り除き (その間に監視があふれてもルート全体は読み直しません)、読み直すのは通知に残ったフォルダーだけです。ハードリンクを数える場合は、更新後にツリー全体をメモリ上で数え直します。不完全または結合されたルートとサポートされていないスコープでは、フォローを開始できません。ネイティブ障害は自動再試行なしで表示されたままになります。共有ファイルを含む部分的な Windows ルートには、外部のエイリアスをカバーするための読み取り可能な USN が必要です。オプトアウトしてタブを閉じてキャンセルし、監視処理の終了を待機します。通知にはソース操作権限は付与されません。独立した Python API は `change_watch.watch` (または `captured_scope` と `watch_scope`) です。ネイティブ CI は、Windows USN および Linux inotify を検証しました (Windows/Linux 外部エイリアス書き込みを含む)。 **オプション → バックグラウンド モニター…** はデフォルトではオフになっています。有効にすると、ウィンドウを閉じるときに FileTree がシステム トレイに保持されます。 **Quit** は常にすべての処理の終了を待ってから終了します。トレイが使用できない場合、ウィンドウは表示されたままになります。トレイを紛失すると、隠されたウィンドウが復元されます。ワーカーは 1 分に 1 回、OS の利用可能な容量を読み取り、サポートされているシステム通知とステータス/ツールチップを通じて、しきい値を下回ったこと (デフォルトでは 10%) を報告します。 OS の設定により通知が抑制される場合があります。未知の容量は未知のままです。有効なローカル履歴に保存される 1 スレッドのジェントル スキャンの場合、最大 32 個のフォルダーと 1 ~ 720 時間の間隔を選択します。スキャンでは、結果のタブとソースはそのまま残ります。不完全な検査範囲は明示的なままです。プロセス全体にわたる永続的なクレームにより、重複した試行が防止されます。失敗/キャンセルされた期間は調整されたままになり、失敗した期間は 1 回結合されます。通常のスキャンや元の項目に対する操作の確認を開始すると、低負荷のバックグラウンド処理をキャンセルし、その終了を待ちます。無効にすると、バックグラウンド処理の終了を待ち、ウィンドウが復元されます。 `py -3 start_file_tree.py --background` は権限昇格を省略し、すでに有効になっている監視と利用可能なトレイのみを非表示にします。フラグがそれを有効にすることはありません。自動クリーンアップは実行されません。コア API `background.MonitorConfig`、`claim_scan`、`due_roots`、および `SpaceWarnings` は独立したままです。 diff --git a/docs/guide_ko.md b/docs/guide_ko.md index b52a3a4d..29d24a6d 100644 --- a/docs/guide_ko.md +++ b/docs/guide_ko.md @@ -76,7 +76,7 @@ FileTree는 사용자가 요청할 때만 스캔합니다. 폴더를 열거나 **디스크 공간이 어디에 있는지 확인하세요.** FileTree는 폴더 또는 전체 드라이브를 검색하고 그 안에 있는 모든 파일을 합산한 다음 가장 큰 폴더와 파일부터 폴더 트리, 다채로운 트리맵 및 가장 큰 파일 목록으로 표시합니다. 더 이상 필요하지 않은 항목을 찾으면 창에서 바로 휴지통으로 이동하세요. -**옵션 → 변경 사항 따르기**(기본적으로 꺼져 있음)에서 각 결과 탭은 전체 물리적 Windows/Linux 스캔을 독립적으로 따릅니다. 백그라운드 네이티브 USN/directory-notification/inotify 피드는 제한된 변경 사항을 1초 동안 모은 뒤, 자체 항목이 바뀐 폴더만 다시 읽고 이미 가진 하위 폴더는 그대로 둡니다. 그래서 스캔한 폴더에 바로 저장된 파일 때문에 스캔 전체를 다시 읽지 않습니다. 변경된 여러 폴더는 한 번에 읽고, 아래에 변경이 매우 많은 폴더는 통째로 다시 읽습니다. 보기의 위치, 펼친 폴더와 선택은 그대로 유지됩니다. 스캔, 분석, 소스 작업 검토, 모달 질문 및 실시간 실행 취소는 새로 고침을 연기합니다. 폴더를 다시 읽는 동안에도 감시는 계속되며, 그 뒤 새 감시에 넘겨줍니다. NTFS 변경 저널은 원래 위치부터 이어서 읽으므로 그 사이의 변경도 놓치지 않습니다. 이벤트가 손실되거나 넘친 경우와 이어 읽을 저널이 없는 소스 작업 공백만 루트 전체를 다시 읽으며, 주기적으로 전체를 다시 읽지는 않습니다. 휴지통으로 이동, 영구 삭제, 실행 취소는 (이 탭이든 다른 탭이든) 항목을 결과에서 바로 제거하고, 알림에 남은 폴더만 다시 읽습니다. 하드 링크를 계산할 때는 업데이트 후 메모리에서 트리 전체를 다시 계산합니다. 불완전하거나 결합된 루트 및 지원되지 않는 범위는 다음을 시작할 수 없습니다. 운영체제 오류는 자동 재시도 없이도 계속 표시됩니다. 공유 파일이 있는 부분 Windows 루트에는 외부 별칭을 포함하기 위해 읽기 가능한 USN이 필요합니다. 옵트아웃 및 탭 닫기를 취소하고 모니터 작업 종료를 기다림하세요. 알림은 소스 작업 권한을 부여하지 않습니다. 독립적인 Python API는 `change_watch.watch`(또는 `captured_scope`와 `watch_scope`)입니다. 운영체제 CI는 Windows/Linux 외부 별칭 쓰기를 포함하여 Windows USN 및 Linux inotify를 확인했습니다. +**옵션 → 변경 사항 따르기**(기본적으로 꺼져 있음)에서 각 결과 탭은 전체 물리적 Windows, Linux 또는 macOS 스캔을 독립적으로 따릅니다. 백그라운드 네이티브 USN/directory-notification/inotify/FSEvents 피드는 제한된 변경 사항을 0.5초 동안 모은 뒤, 자체 항목이 바뀐 폴더만 다시 읽고 이미 가진 하위 폴더는 그대로 둡니다. 그래서 스캔한 폴더에 바로 저장된 파일 때문에 스캔 전체를 다시 읽지 않습니다. 변경된 여러 폴더는 한 번에 읽고, 아래에 변경이 매우 많은 폴더는 통째로 다시 읽습니다. 보기의 위치, 펼친 폴더와 선택은 그대로 유지됩니다. 스캔, 분석, 소스 작업 검토, 모달 질문 및 실시간 실행 취소는 새로 고침을 연기합니다. 폴더를 다시 읽는 동안에도, 이동이나 삭제 같은 작업 중에도 감시는 계속되며, 그 뒤 새 감시에 넘겨줍니다. NTFS 변경 저널은 원래 위치부터 이어서 읽으므로 그 사이의 변경도 놓치지 않습니다. 드라이브 어디에서든 만들어진 하드 링크는 스캔 안에 그 이름이 있는 폴더만 다시 읽습니다. 이벤트가 손실되거나 넘친 경우에만 루트 전체를 다시 읽으며, 주기적으로 전체를 다시 읽지는 않습니다. 휴지통으로 이동, 영구 삭제, 실행 취소는 (이 탭이든 다른 탭이든) 항목을 결과에서 바로 제거하고 (그동안 감시가 넘쳐도 루트 전체를 다시 읽지 않습니다), 알림에 남은 폴더만 다시 읽습니다. 하드 링크를 계산할 때는 업데이트 후 메모리에서 트리 전체를 다시 계산합니다. 불완전하거나 결합된 루트 및 지원되지 않는 범위는 다음을 시작할 수 없습니다. 운영체제 오류는 자동 재시도 없이도 계속 표시됩니다. 공유 파일이 있는 부분 Windows 루트에는 외부 별칭을 포함하기 위해 읽기 가능한 USN이 필요합니다. 옵트아웃 및 탭 닫기를 취소하고 모니터 작업 종료를 기다림하세요. 알림은 소스 작업 권한을 부여하지 않습니다. 독립적인 Python API는 `change_watch.watch`(또는 `captured_scope`와 `watch_scope`)입니다. 운영체제 CI는 Windows/Linux 외부 별칭 쓰기를 포함하여 Windows USN 및 Linux inotify를 확인했습니다. **옵션 → 백그라운드 모니터…**은 기본적으로 꺼져 있습니다. 창을 닫을 때 시스템 트레이에 FileTree를 유지하려면 이를 활성화하십시오. **Quit**은 항상 모든 작업이 끝날 때까지 기다린 후 종료합니다. 트레이를 사용할 수 없는 경우 창은 계속 표시됩니다. 트레이를 잃어버리면 숨겨진 창이 복원됩니다. 1분에 한 번씩 작업자는 OS의 사용 가능한 용량을 읽고 지원되는 시스템 알림 및 상태/도구 설명을 통해 임계값 미만 초과(기본적으로 10%)를 보고합니다. OS 기본 설정은 알림을 표시하지 않을 수 있습니다. 알 수 없는 용량은 알 수 없는 상태로 유지됩니다. 활성화된 로컬 기록에 저장되는 단일 스레드 젠틀 스캔에 대해 최대 32개의 폴더와 1~720시간 간격을 선택합니다. 스캔 결과 탭과 소스는 그대로 유지됩니다. 불완전한 적용 범위는 명시적으로 유지됩니다. 프로세스 전반에 걸친 영구 실행 기록은 중복 시도를 방지합니다. 실패/취소된 기간은 제한된 상태로 유지되고 누락된 기간은 한 번 통합됩니다. 일반 스캔이나 원본 작업 검토를 시작하면 저부하 백그라운드 작업을 취소하고 종료를 기다립니다. 비활성화하면 백그라운드 작업이 끝날 때까지 기다리고 창이 복원됩니다. `py -3 start_file_tree.py --background`은 권한 상승을 생략하고 이미 활성화된 모니터링과 사용 가능한 트레이에서만 숨깁니다. 플래그는 이를 활성화하지 않습니다. 자동 정리는 수행되지 않습니다. 핵심 API `background.MonitorConfig`, `claim_scan`, `due_roots` 및 `SpaceWarnings`은 독립적으로 유지됩니다. diff --git a/docs/guide_zh-CN.md b/docs/guide_zh-CN.md index 39ec6c2b..a7637004 100644 --- a/docs/guide_zh-CN.md +++ b/docs/guide_zh-CN.md @@ -76,7 +76,7 @@ FileTree 只在要求时才扫描:打开或拖入文件夹、点击“重新 **看看磁盘空间都用到哪里去了。** FileTree 会扫描一个文件夹或整个磁盘,把里面每个文件的大小加总,先列出最大的文件夹和文件——用文件夹树、彩色的方块图和最大文件列表呈现。找到不需要的东西时,直接在窗口里把它移到回收站。 -**选项 → 跟随变更** 默认关闭,每个结果选项卡可独立跟随完整、实际的 Windows/Linux 扫描。后台原生 USN/目录通知/inotify 将数量受限的变化合并一秒,之后只重新读取自身项目有变化的文件夹,并保留其中已有的子文件夹;因此直接存进扫描文件夹的文件不会让整个扫描重扫。多个变化的文件夹一次读完;下面变化非常多的文件夹则整个重读。视图会保留原来的位置、展开的文件夹与选择。扫描、分析、来源操作审核、询问窗口及有效撤销提示期间延后更新。读取文件夹期间监看持续进行,之后交接给新的监看;NTFS 变更日志会从原来的位置继续读,中间的变化不会漏掉。只有事件丢失或溢出,以及没有变更日志可接续的来源操作间隔,才会重新读取整个根目录;不会定时完整重扫。移到回收站、永久删除与撤销(无论在这个还是其他选项卡)会直接从结果移除项目,只重新读取提示栏中留下的文件夹。计入硬链接时,更新后在内存中重新计算整棵树。不完整、合并或不支持的范围不能启用;原生错误持续显示,不自动重试。Windows 部分范围含共享文件时须可读取 USN,才能涵盖范围外别名。关闭选项或选项卡会取消并等待监看任务结束。通知不授予来源操作权限。独立 Python API 为 `change_watch.watch`(或 `captured_scope` 与 `watch_scope`);原生 CI 已验证 Windows USN 与 Linux inotify,包含 Windows/Linux 范围外别名写入。 +**选项 → 跟随变更** 默认关闭,每个结果选项卡可独立跟随完整、实际的 Windows、Linux 或 macOS 扫描。后台原生 USN/目录通知/inotify/FSEvents 将数量受限的变化合并半秒,之后只重新读取自身项目有变化的文件夹,并保留其中已有的子文件夹;因此直接存进扫描文件夹的文件不会让整个扫描重扫。多个变化的文件夹一次读完;下面变化非常多的文件夹则整个重读。视图会保留原来的位置、展开的文件夹与选择。扫描、分析、来源操作审核、询问窗口及有效撤销提示期间延后更新。读取文件夹期间,以及移动、删除等操作期间,监看都持续进行,之后交接给新的监看;NTFS 变更日志会从原来的位置继续读,中间的变化不会漏掉。磁盘上任何地方建立的硬链接,只会重新读取扫描中含有其名称的文件夹。只有事件丢失或溢出才会重新读取整个根目录;不会定时完整重扫。移到回收站、永久删除与撤销(无论在这个还是其他选项卡)会直接从结果移除项目(期间监看溢出也不会重读整个根目录),只重新读取提示栏中留下的文件夹。计入硬链接时,更新后在内存中重新计算整棵树。不完整、合并或不支持的范围不能启用;原生错误持续显示,不自动重试。Windows 部分范围含共享文件时须可读取 USN,才能涵盖范围外别名。关闭选项或选项卡会取消并等待监看任务结束。通知不授予来源操作权限。独立 Python API 为 `change_watch.watch`(或 `captured_scope` 与 `watch_scope`);原生 CI 已验证 Windows USN 与 Linux inotify,包含 Windows/Linux 范围外别名写入。 **选项 → 后台监视器…** 默认关闭。启用后,关闭窗口会将 FileTree 保留在托盘;**退出** 一定等待全部任务并结束。没有托盘时窗口保持显示,托盘消失也会恢复隐藏窗口。后台任务每分钟读取系统可用容量,低于阈值时(默认 10%)通过支持的系统通知与状态/提示显示;系统偏好设置可能停用通知,未知容量保持未知。可选最多 32 个文件夹及 1–720 小时间隔,以单个低优先级线程扫描并保存至已启用的本地历史。扫描不改动结果选项卡或来源,不完整范围保持明确。跨进程持久记录防止重复尝试;失败/取消的期间不重跑,错过多次计划只合并一次。前台扫描/来源审核会取消并等待低优先级任务。停用也等待任务并恢复窗口。`py -3 start_file_tree.py --background` 不询问提升权限,仅在监视已启用且托盘可用时隐藏;标志不会启用监视。不自动清理。核心 API `background.MonitorConfig`、`claim_scan`、`due_roots` 及 `SpaceWarnings` 保持独立。 diff --git a/docs/guide_zh-TW.md b/docs/guide_zh-TW.md index 0b4c6b77..7b258e08 100644 --- a/docs/guide_zh-TW.md +++ b/docs/guide_zh-TW.md @@ -76,7 +76,7 @@ FileTree 只在要求時才掃描:開啟或拖入資料夾、按「重新掃 **看看磁碟空間都用到哪裡去了。** FileTree 會掃描一個資料夾或整顆磁碟,把裡面每個檔案的大小加總,先列出最大的資料夾和檔案——用資料夾樹、彩色的方塊圖和最大檔案清單呈現。找到不需要的東西時,直接在視窗裡把它移到資源回收筒。 -**選項 → 跟隨變更** 預設關閉,每個結果分頁可獨立跟隨完整、實際的 Windows/Linux 掃描。背景原生 USN/目錄通知/inotify 將數量受限的變動合併一秒,之後只重新讀取自身項目有變動的資料夾,並保留其中已有的子資料夾;因此直接存進掃描資料夾的檔案不會讓整個掃描重掃。多個變動的資料夾一次讀完;底下變動非常多的資料夾則整個重讀。檢視會保留原本的位置、展開的資料夾與選取。掃描、分析、來源操作審查、詢問視窗及有效復原提示期間延後更新。讀取資料夾期間監看持續進行,之後交接給新的監看;NTFS 變更日誌會從原本的位置繼續讀,中間的變動不會漏掉。只有事件遺失或溢位,以及沒有變更日誌可接續的來源操作間隔,才會重新讀取整個根目錄;不會定時完整重掃。移到資源回收筒、永久刪除與復原(不論在這個或其他分頁)會直接從結果移除項目,只重新讀取提示列中留下的資料夾。計入硬連結時,更新後在記憶體中重新計算整棵樹。不完整、合併或不支援的範圍不能啟用;原生錯誤持續顯示,不自動重試。Windows 部分範圍含共用檔案時須可讀取 USN,才能涵蓋範圍外別名。關閉選項或分頁會取消並等待監看工作結束。通知不授予來源操作權限。獨立 Python API 為 `change_watch.watch`(或 `captured_scope` 與 `watch_scope`);原生 CI 已驗證 Windows USN 與 Linux inotify,包含 Windows/Linux 範圍外別名寫入。 +**選項 → 跟隨變更** 預設關閉,每個結果分頁可獨立跟隨完整、實際的 Windows、Linux 或 macOS 掃描。背景原生 USN/目錄通知/inotify/FSEvents 將數量受限的變動合併半秒,之後只重新讀取自身項目有變動的資料夾,並保留其中已有的子資料夾;因此直接存進掃描資料夾的檔案不會讓整個掃描重掃。多個變動的資料夾一次讀完;底下變動非常多的資料夾則整個重讀。檢視會保留原本的位置、展開的資料夾與選取。掃描、分析、來源操作審查、詢問視窗及有效復原提示期間延後更新。讀取資料夾期間,以及移動、刪除等操作期間,監看都持續進行,之後交接給新的監看;NTFS 變更日誌會從原本的位置繼續讀,中間的變動不會漏掉。磁碟上任何地方建立的硬連結,只會重新讀取掃描中含有其名稱的資料夾。只有事件遺失或溢位才會重新讀取整個根目錄;不會定時完整重掃。移到資源回收筒、永久刪除與復原(不論在這個或其他分頁)會直接從結果移除項目(期間監看溢位也不會重讀整個根目錄),只重新讀取提示列中留下的資料夾。計入硬連結時,更新後在記憶體中重新計算整棵樹。不完整、合併或不支援的範圍不能啟用;原生錯誤持續顯示,不自動重試。Windows 部分範圍含共用檔案時須可讀取 USN,才能涵蓋範圍外別名。關閉選項或分頁會取消並等待監看工作結束。通知不授予來源操作權限。獨立 Python API 為 `change_watch.watch`(或 `captured_scope` 與 `watch_scope`);原生 CI 已驗證 Windows USN 與 Linux inotify,包含 Windows/Linux 範圍外別名寫入。 **選項 → 背景監視器…** 預設關閉。啟用後,關閉視窗會將 FileTree 保留在系統匣;**結束** 一定等待全部工作並退出。沒有系統匣時視窗保持顯示,系統匣消失也會恢復隱藏視窗。背景工作每分鐘讀取系統可用容量,低於門檻時(預設 10%)透過支援的系統通知與狀態/提示顯示;系統偏好設定可能停用通知,未知容量維持未知。可選最多 32 個資料夾及 1–720 小時間隔,以單一低優先順序執行緒掃描並儲存至已啟用的本機歷史。掃描不改動結果分頁或來源,不完整範圍保持明確。跨程序持久記錄防止重複嘗試;失敗/取消的期間不重跑,錯過多次排程只合併一次。前景掃描/來源審查會取消並等待低優先順序工作。停用也等待工作並恢復視窗。`py -3 start_file_tree.py --background` 不詢問提升權限,僅在監視已啟用且系統匣可用時隱藏;旗標不會啟用監視。不自動清理。核心 API `background.MonitorConfig`、`claim_scan`、`due_roots` 及 `SpaceWarnings` 保持獨立。 diff --git a/docs/updates/2026-11.md b/docs/updates/2026-11.md index c7c3d7bd..b50c7bd3 100644 --- a/docs/updates/2026-11.md +++ b/docs/updates/2026-11.md @@ -762,3 +762,24 @@ - **未驗證**:Linux inotify 新舊監看並存與監看數量上限;使用者實際資料夾上的長時間使用。 - **檔案**:`je_file_tree/gui/fluent_icons.py`、`gui/folder_refresh.py`;`test/test_fluent_icons.py`、`test/test_change_watch_gui.py`;`progress.md`。 - **待辦**:`progress.md` #144(已移除 USN 部分)。 + +## U-20261120-03 · 2026-11-20 · 「跟隨變更」在操作期間不中斷、硬連結只讀相關資料夾,並支援 macOS · #feature #perf #gui #macos + +- **原因**:使用者要求處理 U-20261120-02 之後列出的剩餘問題。 +- **變更**: + - **來源操作期間不再停止監看**:以前移動、重新命名、壓縮等審查與操作開始前會停止監看,結束後若沒有變更日誌可接續,就重掃整個根目錄(即使審查被取消)。監看不讀取樹,也沒有開著會阻擋項目移動的控制代碼,所以現在只停止讀取樹的範圍擷取,操作期間的變動照常回報,結束後只重新讀取變動的資料夾,並重新擷取範圍。操作期間若監看溢位,仍視為操作的空窗:移到資源回收筒、永久刪除與復原會撤回它(與以前相同),其他操作結束後重讀整個根目錄。 + - **硬連結**:以前磁碟區上任何地方建立或刪除硬連結都會整個重掃。現在以檔案編號(`OpenFileById`、`FindFirstFileNameW`)查出該檔案的所有名稱,只重新讀取掃描範圍內含有其名稱的資料夾;一次讀取超過 64 個、或查詢被拒時才整個重掃。 + - **變更日誌連續讀取**:以前每 0.25 秒只讀 32 KB,磁碟很忙時會落後;現在還有待讀記錄就立刻接著讀(中間讓路給視窗)。 + - **目錄通知緩衝區**:本機磁碟由 32 KB 加大為 256 KB(網路磁碟 64 KB,系統上限),減少溢位。 + - **反應時間**:計時器由 1 秒改為 0.25 秒,變動合併由 1 秒改為 0.5 秒;狀態文字只在改變時更新。 + - **inotify 的空窗**:新監看就緒後,它新監看的資料夾會再讀一次,補上就緒前的變動。 + - **macOS**:新增 `core/mac_watch.py`,以 FSEvents 監看掃描資料夾。FSEvents 以資料夾回報,正好對應只讀變動的資料夾;它明確回報事件遺失(整個重讀)、跟不上(該資料夾整個重讀)、根目錄移動與磁碟掛載變更(整個重讀)。和目錄通知一樣,含硬連結檔案的部分範圍不能啟用,因為範圍外的名稱不會回報。 + - 日文與韓文的「不支援」字串原本是錯誤的機器翻譯,日文的 inotify 名稱被譯成「通知する」,一併修正。 +- **驗證**: + - 新增原生測試:在掃描範圍外為範圍內的檔案建立硬連結,本機以 NTFS 變更日誌執行,只回報該檔案所在的資料夾,沒有整個重掃。 + - 新增:操作期間監看持續回報、審查結束後只讀變動的資料夾、操作期間溢位的撤回與保留、inotify 交接後重讀新監看的資料夾、名稱查詢(範圍內、範圍外、已刪除、被拒、超過上限)、變更日誌連續讀取、緩衝區大小、FSEvents 事件對應。原本預期「操作前停止監看」的 4 個測試,改為卡住範圍擷取來驗證操作仍會等它。 + - 修正一個偶爾失敗的測試替身(先等取消最多 5 秒,與等待逾時競爭);修正後連續 6 次通過。 + - 相關的 61 個測試檔案共 880 個測試通過(排除 1 個已知的長路徑測試,略過 9 個);`ruff check .` 通過。 +- **未驗證**:macOS FSEvents 只能在 CI 的 macOS 工作上原生執行,本機沒有 Mac;Linux inotify 交接只由 CI 涵蓋。 +- **檔案**:`je_file_tree/core/mac_watch.py`(新)、`core/change_watch.py`、`core/windows_watch.py`;`je_file_tree/gui/change_watch.py`、`gui/strings.py`;`architecture.md`、`docs/core-api.md`、`docs/guide.md` 與四個翻譯;`progress.md`;測試檔案。 +- **待辦**:`progress.md` #144。 diff --git a/docs/updates/README.md b/docs/updates/README.md index 02f48ff2..5cd7cd30 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -70,6 +70,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261120-03 | 2026-11-20 | 「跟隨變更」在操作期間不中斷、硬連結只讀相關資料夾,並支援 macOS | #feature #perf #gui #macos | [2026-11](2026-11.md) | | U-20261120-02 | 2026-11-20 | 在本機原生驗證「跟隨變更」只讀變動資料夾,並修正兩處多餘工作 | #validation #perf #gui | [2026-11](2026-11.md) | | U-20261120-01 | 2026-11-20 | 「跟隨變更」只重新讀取變動的資料夾,不再重掃整個掃描 | #perf #gui #scan | [2026-11](2026-11.md) | | U-20261119-07 | 2026-11-19 | 大型資料夾的加總、排序與清理比對也分頁讓路給視窗 | #perf #fix | [2026-11](2026-11.md) | diff --git a/je_file_tree/core/change_watch.py b/je_file_tree/core/change_watch.py index a8e06187..fdfa1c8a 100644 --- a/je_file_tree/core/change_watch.py +++ b/je_file_tree/core/change_watch.py @@ -181,5 +181,9 @@ def watch_scope(scope: WatchScope, on_change: Callable[[ChangeBatch], None], can from je_file_tree.core.linux_watch import watch_linux # noqa: PLC0415 - native backend watch_linux(folders, on_change, cancel, ready, scope.shared) + elif sys.platform == "darwin": + from je_file_tree.core.mac_watch import watch_mac # noqa: PLC0415 - native backend + + watch_mac(folders, on_change, cancel, ready, scope.shared) else: - raise ValueError("Follow changes is available on Windows and Linux") + raise ValueError("Follow changes is available on Windows, Linux and macOS") diff --git a/je_file_tree/core/mac_watch.py b/je_file_tree/core/mac_watch.py new file mode 100644 index 00000000..b06167db --- /dev/null +++ b/je_file_tree/core/mac_watch.py @@ -0,0 +1,140 @@ +"""One FSEvents stream over the scanned folder, reporting the folders whose entries changed and every loss. + +FSEvents reports directories, not files: each path is a folder whose own entries changed, which is +what the window reads again. The stream says when it could not keep up (``MustScanSubDirs``: that +folder is read whole), when events were dropped or their numbers wrapped (the whole root is read), +and when the scanned folder itself was moved or a volume was mounted or unmounted inside it. +Read-only: no file is opened and nothing is changed. Called through ``change_watch.watch_scope``. +""" + +from collections.abc import Callable +import ctypes +import ctypes.util +import os +import posixpath +import threading + +from je_file_tree.core.change_watch import ChangeBatch, POLL_SECONDS, WatchFile, WatchFolder, changed, check_folder + +_MUST_SCAN_SUBDIRS = 0x1 +_LOST = 0x2 | 0x4 | 0x8 # user dropped, kernel dropped, event numbers wrapped +_SCOPE = 0x20 | 0x40 | 0x80 # root changed, mount, unmount +_NO_DEFER, _WATCH_ROOT = 0x2, 0x4 +_SINCE_NOW = 0xFFFFFFFFFFFFFFFF +_UTF8 = 0x08000100 +_LATENCY = 0.2 # s FSEvents gathers events for; the window coalesces them further +_DATA_VOLUME = "/System/Volumes/Data" # where FSEvents may report the paths of the data volume + +_Callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, + ctypes.POINTER(ctypes.c_char_p), ctypes.POINTER(ctypes.c_uint32), + ctypes.POINTER(ctypes.c_uint64)) + + +def spellings(root: str) -> tuple[str, ...]: + """The forms FSEvents may report ``root`` in: as scanned, resolved, and on the data volume.""" + real = os.path.realpath(root) + forms = [root, real] + if not real.startswith(_DATA_VOLUME + "/"): + forms.append(_DATA_VOLUME + real) + return tuple(dict.fromkeys(form.rstrip("/") or "/" for form in forms)) + + +def _inside(path: str, root: str, forms: tuple[str, ...]) -> str | None: + path = path.rstrip("/") or "/" + for form in forms: + if path == form: + return root + base = "" if form == "/" else form + if path.startswith(base + "/"): + return posixpath.join(root, path[len(base) + 1:]) + return None + + +def parse_fsevents(paths: list[str], flags: list[int], root: str, forms: tuple[str, ...]) -> ChangeBatch: + """Map one delivery of directory events to folders inside ``root``; a loss asks for all of it.""" + folders, trees = set(), set() + for path, flag in zip(paths, flags, strict=True): + if flag & _LOST: + return changed(set(), full=True, reason="notification_lost") + inside = _inside(path, root, forms) + if flag & _SCOPE and (inside is not None or flag & 0x20): + return changed(set(), full=True, reason="scope_changed") + if inside is not None: + (trees if flag & _MUST_SCAN_SUBDIRS else folders).add(inside) + return changed(folders, trees=trees) + + +def _libraries(): + core = ctypes.CDLL(ctypes.util.find_library("CoreFoundation")) + services = ctypes.CDLL(ctypes.util.find_library("CoreServices")) + core.CFStringCreateWithCString.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_uint32] + core.CFStringCreateWithCString.restype = ctypes.c_void_p + core.CFArrayCreate.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p), ctypes.c_long, ctypes.c_void_p] + core.CFArrayCreate.restype = ctypes.c_void_p + core.CFRelease.argtypes = [ctypes.c_void_p] + core.CFRunLoopGetCurrent.restype = ctypes.c_void_p + core.CFRunLoopRunInMode.argtypes = [ctypes.c_void_p, ctypes.c_double, ctypes.c_bool] + core.CFRunLoopRunInMode.restype = ctypes.c_int32 + services.FSEventStreamCreate.argtypes = [ctypes.c_void_p, _Callback, ctypes.c_void_p, ctypes.c_void_p, + ctypes.c_uint64, ctypes.c_double, ctypes.c_uint32] + services.FSEventStreamCreate.restype = ctypes.c_void_p + for name in ("FSEventStreamScheduleWithRunLoop",): + getattr(services, name).argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] + for name in ("FSEventStreamStop", "FSEventStreamInvalidate", "FSEventStreamRelease"): + getattr(services, name).argtypes = [ctypes.c_void_p] + services.FSEventStreamStart.argtypes = [ctypes.c_void_p] + services.FSEventStreamStart.restype = ctypes.c_bool + return core, services + + +def watch_mac(folders: tuple[WatchFolder, ...], on_change: Callable[[ChangeBatch], None], + cancel: threading.Event, ready: Callable[[str], None] | None, + shared: tuple[WatchFile, ...] = ()) -> None: + """Run one stream on this thread's run loop until ``cancel``; native failures raise visibly.""" + root = folders[0].path + if shared and not os.path.ismount(root): + raise ValueError("External hard-link changes are not reported on macOS; scan the whole volume") + core, services = _libraries() + forms = spellings(root) + failures: list[BaseException] = [] + + def delivered(_stream, _info, count, paths, flags, _numbers) -> None: + try: + names = [os.fsdecode(paths[index]) for index in range(count)] + batch = parse_fsevents(names, [flags[index] for index in range(count)], root, forms) + if (batch.folders or batch.trees or batch.full) and not cancel.is_set(): + on_change(batch) + except (ValueError, UnicodeError) as error: # handed to this thread's loop, which raises it + failures.append(error) + + callback = _Callback(delivered) # kept alive for as long as the stream may call it + text = core.CFStringCreateWithCString(None, os.fsencode(root), _UTF8) + if not text: + raise OSError("FSEvents could not name the scanned folder") + array = core.CFArrayCreate(None, (ctypes.c_void_p * 1)(text), 1, + ctypes.addressof(ctypes.c_void_p.in_dll(core, "kCFTypeArrayCallBacks"))) + stream = services.FSEventStreamCreate(None, callback, None, array, _SINCE_NOW, _LATENCY, _NO_DEFER | _WATCH_ROOT) + if not stream: + core.CFRelease(array) + core.CFRelease(text) + raise OSError("FSEvents stream unavailable") + mode = ctypes.c_void_p.in_dll(core, "kCFRunLoopDefaultMode").value + try: + services.FSEventStreamScheduleWithRunLoop(stream, core.CFRunLoopGetCurrent(), mode) + if not services.FSEventStreamStart(stream): + raise OSError("FSEvents stream did not start") + if ready: + ready("fsevents") + try: + while not cancel.is_set() and not failures: + check_folder(folders[0]) + core.CFRunLoopRunInMode(mode, POLL_SECONDS, False) + finally: + services.FSEventStreamStop(stream) + if failures: + raise failures[0] + finally: + services.FSEventStreamInvalidate(stream) + services.FSEventStreamRelease(stream) + core.CFRelease(array) + core.CFRelease(text) diff --git a/je_file_tree/core/windows_watch.py b/je_file_tree/core/windows_watch.py index 713ad72c..38d0b26c 100644 --- a/je_file_tree/core/windows_watch.py +++ b/je_file_tree/core/windows_watch.py @@ -11,10 +11,17 @@ POLL_SECONDS, ChangeBatch, Position, WatchFile, WatchFolder, changed, check_folder, ) from je_file_tree.core.compression_ops import _kernel +from je_file_tree.core.pacing import give_way from je_file_tree.core.windows_allocation import _FileID, _Identity from je_file_tree.core.windows_trash import _volume _BUFFER = 32768 +# ReadDirectoryChangesW keeps changes for its handle in a buffer this size; when a burst fills it, all +# of them are lost and the whole root is read again. Over a network it refuses more than 64 KiB. +_NOTIFY_LOCAL, _NOTIFY_REMOTE = 262144, 65536 +_DRIVE_REMOTE = 4 +_MAX_LINKED = 64 # files whose other names changed, resolved per journal read; past it, read the whole root +_GONE = (2, 3, 87) # not found, path not found, invalid parameter (a file number no longer in use) _NOTIFY = struct.Struct("III") _USN = struct.Struct(" ChangeBa """ if not payload: return changed(set(), full=True, reason="notification_lost") - if len(payload) > _BUFFER: + if len(payload) > _NOTIFY_LOCAL: raise ValueError("Oversized Windows notification buffer") offset, paths = 0, set() while offset < len(payload): @@ -72,9 +79,26 @@ def parse_notifications(payload: bytes, root: str, scopes: set[str]) -> ChangeBa raise ValueError("Unterminated Windows notification chain") -def parse_usn(payload: bytes, scopes: dict[int, str], root: str, start: int, - shared: dict[int, tuple[str, ...]] | None = None) -> tuple[int, ChangeBatch]: - """Accept bounded V2 records/cursors; unknown volume parents are outside the captured map.""" +def _relinked(inode: int, parent: int, scopes: dict[int, str], shared: dict[int, tuple[str, ...]] | None, + linked: set[int] | None) -> bool: + """Note a file whose names changed; True when, unable to look its names up, the whole root is needed.""" + if shared is not None and inode in shared: + return False # its captured names' folders are read + if linked is not None: + linked.add(inode) # its other names, in the scan or not, are looked up + return False + return parent not in scopes + + +def parse_usn(payload: bytes, scopes: dict[int, str], root: str, start: int, # noqa: PLR0913 + shared: dict[int, tuple[str, ...]] | None = None, linked: set[int] | None = None + ) -> tuple[int, ChangeBatch]: + """Accept bounded V2 records/cursors; unknown volume parents are outside the captured map. + + A file that gained or lost a name outside the scan may still have one inside it. With ``linked``, + such files are collected there for the caller to look up (``_linked_folders``); without it, the + whole root is asked for. + """ if not _CURSOR_BYTES <= len(payload) <= _BUFFER: raise ValueError("Invalid USN reply size") next_usn, = struct.unpack_from(" i return None +def _names_of(kernel, volume, number: int) -> list[str]: + """Every name of file ``number`` on ``volume``, relative to the volume's root (metadata only).""" + descriptor = _FileIdDescriptor(ctypes.sizeof(_FileIdDescriptor), 0, number, 0) + handle = kernel.OpenFileById(volume, ctypes.byref(descriptor), 0x80, 7, None, 0x02200000) + if handle in (None, ctypes.c_void_p(-1).value): + raise ctypes.WinError(ctypes.get_last_error()) + text = ctypes.create_unicode_buffer(32768) + try: + length = kernel.GetFinalPathNameByHandleW(handle, text, len(text), 0) + if not length or length >= len(text): + raise ctypes.WinError(ctypes.get_last_error()) + finally: + kernel.CloseHandle(handle) + names, size = [], ctypes.c_uint32(len(text)) + found = kernel.FindFirstFileNameW(text.value, 0, ctypes.byref(size), text) + if found in (None, ctypes.c_void_p(-1).value): + raise ctypes.WinError(ctypes.get_last_error()) + try: + names.append(text.value) + while True: + size = ctypes.c_uint32(len(text)) + if not kernel.FindNextFileNameW(found, ctypes.byref(size), text): + break + names.append(text.value) + finally: + kernel.FindClose(found) + return names + + +def _linked_folders(kernel, volume, numbers: set[int], drive: str, root: str) -> set[str] | None: + """The folders inside ``root`` holding a name of each file in ``numbers``; None when one cannot be told. + + A hard link made or removed anywhere on the volume used to read the whole scan again (Windows + servicing makes many); now only a file with a name inside the scan reads that name's folder. + """ + if len(numbers) > _MAX_LINKED: + return None + base = ntpath.normcase(root.rstrip("\\")) + found = set() + for number in numbers: + try: + names = _names_of(kernel, volume, number) + except OSError as error: + if getattr(error, "winerror", None) in _GONE: + continue # no name left: those in the scan went with records of their own folders + return None + for name in names: + path = ntpath.join(drive, name.lstrip("\\")) + folder = ntpath.normcase(path) + if folder.startswith(base + "\\"): + found.add(ntpath.dirname(path)) + return found + + +def _with_linked(kernel, volume, batch: ChangeBatch, linked: set[int], drive: str, root: str # noqa: PLR0913 + ) -> ChangeBatch: + """``batch`` with the folders holding a name of each file in ``linked`` (all of the root if unknown).""" + if not linked or batch.full: + return batch + elsewhere = _linked_folders(kernel, volume, linked, drive, root) + if elsewhere is None: + return changed(set(), full=True, reason="hard_links_changed") + return changed(set(batch.folders) | elsewhere, trees=set(batch.trees)) + + def _watch_usn(kernel, folders, on_change, cancel, ready, shared, *, since=None, moved=None) -> None: # noqa: PLR0913 + moved = moved or (lambda _position: None) root, identifier = _volume(folders[0].path) scopes = {folder.inode: folder.path for folder in folders} aliases = {file.inode: file.parents for file in shared} @@ -183,12 +291,18 @@ def _watch_usn(kernel, folders, on_change, cancel, ready, shared, *, since=None, on_change(changed(set(), full=True, reason="journal_changed")) return payload = _control(kernel, handle, _READ_USN, _READ.pack(cursor, 0xFFFFFFFF, 0, 0, 0, journal)) - cursor, batch = parse_usn(payload, scopes, folders[0].path, cursor, aliases) - if moved: - moved((journal, cursor)) - if (batch.folders or batch.full) and not cancel.is_set(): + started, linked = cursor, set() + cursor, batch = parse_usn(payload, scopes, folders[0].path, cursor, aliases, linked) + batch = _with_linked(kernel, handle, batch, linked, root, folders[0].path) + moved((journal, cursor)) + if (batch.folders or batch.trees or batch.full) and not cancel.is_set(): on_change(batch) - if batch.full or cancel.wait(POLL_SECONDS): + if batch.full: + return + if started < cursor < next_usn: + give_way() # more records wait: read on now, so a busy volume never leaves the feed behind + continue + if cancel.wait(POLL_SECONDS): return finally: kernel.CloseHandle(handle) @@ -230,9 +344,17 @@ def _completion(kernel, handle, overlap: _Overlapped, cancel: threading.Event) - return 0 -def _notification(kernel, handle, overlap: _Overlapped, cancel: threading.Event, - armed: Callable[[], None] | None = None) -> bytes: - buffer = ctypes.create_string_buffer(_BUFFER) +def _notify_size(kernel, path: str) -> int: + """The largest notification buffer ReadDirectoryChangesW takes for ``path``.""" + if path.startswith("\\\\"): + return _NOTIFY_REMOTE + drive = ntpath.splitdrive(path)[0] + "\\" + return _NOTIFY_REMOTE if kernel.GetDriveTypeW(drive) == _DRIVE_REMOTE else _NOTIFY_LOCAL + + +def _notification(kernel, handle, overlap: _Overlapped, cancel: threading.Event, # noqa: PLR0913 + armed: Callable[[], None] | None = None, size: int = _NOTIFY_LOCAL) -> bytes: + buffer = ctypes.create_string_buffer(size) kernel.ResetEvent(overlap.event) if (not kernel.ReadDirectoryChangesW(handle, buffer, len(buffer), True, 0x15F, None, ctypes.byref(overlap), None) and ctypes.get_last_error() != _PENDING): @@ -261,12 +383,13 @@ def _watch_notifications(kernel, folders, on_change, cancel, ready) -> None: try: overlap = _Overlapped(event=event) scopes = {folder.path for folder in folders} + size = _notify_size(kernel, folders[0].path) first = True while not cancel.is_set(): check_folder(folders[0]) try: armed = (lambda: ready("directory_changes")) if first and ready else None - payload = _notification(kernel, handle, overlap, cancel, armed) + payload = _notification(kernel, handle, overlap, cancel, armed, size) first = False batch = parse_notifications(payload, folders[0].path, scopes) except OSError as error: diff --git a/je_file_tree/gui/change_watch.py b/je_file_tree/gui/change_watch.py index c0e26f09..66ae2844 100644 --- a/je_file_tree/gui/change_watch.py +++ b/je_file_tree/gui/change_watch.py @@ -19,7 +19,9 @@ from PySide6.QtCore import QObject, QThread, Qt, QTimer from PySide6.QtWidgets import QApplication, QLabel -from je_file_tree.core.change_watch import ChangeBatch, Position, WatchScope, captured_scope, merged, watch_scope +from je_file_tree.core.change_watch import ( + ChangeBatch, Position, WatchScope, captured_scope, changed, merged, watch_scope, +) from je_file_tree.core.node import Node, folder_holding from je_file_tree.core.scanner import EXCLUDED from je_file_tree.gui.i18n import tr @@ -30,12 +32,13 @@ from je_file_tree.gui.main_window import MainWindow ENABLED_KEY = "follow_changes" -COALESCE_SECONDS = 1.0 +COALESCE_SECONDS = 0.5 # changes this close together are read in one go; analysing after each also spaces them +TICK_MS = 250 def supported() -> bool: - """Native change feeds currently support Windows and Linux.""" - return sys.platform == "win32" or sys.platform.startswith("linux") + """Native change feeds: NTFS journal or directory notifications, inotify, FSEvents.""" + return sys.platform in ("win32", "darwin") or sys.platform.startswith("linux") class CaptureWorker(QThread): @@ -137,7 +140,9 @@ def __init__(self, window: MainWindow) -> None: self.root: Node | None = None self.enabled, self._closing = False, False self._pending = ChangeBatch() - self._operation_gap = False # the feed was stopped for a source operation: read the whole root again + self._operation_gap = False # a feed was lost during a source operation: read the whole root again + self._operating = False # a source operation runs (``quiesce``): a lost feed counts as its gap + self._stale = False # the scope was not captured again since an operation: capture it when idle self._handoff: Position | None = None # where the next journal feed reads on from self._since = 0.0 self._status, self._detail = "follow_waiting", "" @@ -148,7 +153,7 @@ def __init__(self, window: MainWindow) -> None: self.label.hide() window.statusBar().addPermanentWidget(self.label) self.timer = QTimer(self) - self.timer.setInterval(1000) + self.timer.setInterval(TICK_MS) self.timer.timeout.connect(self.tick) def configure(self, enabled: bool) -> None: @@ -271,7 +276,7 @@ def stop(self, *, clear: bool = False, wait: bool = False) -> tuple[QThread, ... if clear: self.root = None self._pending, self._since = ChangeBatch(), 0.0 - self._operation_gap = False + self._operation_gap = self._operating = self._stale = False self._handoff = None retiring = tuple(self._retiring) if wait: @@ -280,32 +285,35 @@ def stop(self, *, clear: bool = False, wait: bool = False) -> tuple[QThread, ... return retiring def quiesce(self, *, wait: bool = True) -> tuple[QThread, ...]: - """Stop tree capture before source operations; their observation gap requires full reconciliation. + """Stop reading the tree before a source operation changes it; the feed keeps reporting. - A journal feed's position is kept: the next one reads on from it, which closes that gap. + Only the capture reads the tree, so only it is stopped (the operation waits for what this + returns); it runs again once the operation is over, which may have added or removed folders. + The feed reads none of the tree and holds no handle that keeps an entry from being moved or + removed, so what the operation and any other program change meanwhile is reported and read + again afterwards: no gap is left, and nothing asks for the whole root (U-20261120-03). """ - active = self.worker is not None - if active: - self.hold() - retiring = self.stop(wait=wait) - if active: - if not self._operation_gap and not self._pending.full and not self._pending.folders: - self._since = time.monotonic() - self._operation_gap = True + capture, self._capture = self._capture, None + if capture is not None: + self._retire(capture) + self._operating = self._stale = True + retiring = tuple(worker for worker in self._retiring if isinstance(worker, CaptureWorker)) + if wait: + for pending in retiring: + wait_for(pending) return retiring def resume(self) -> None: - """Follow again after an operation whose changes the results already show or have on offer. + """Follow on after an operation whose changes the results already show or have on offer. - Stopping the feed for a source operation asks for the whole root to be read again, because - nothing was observed meanwhile, unless a journal feed reads on from where it stopped. A move to - the Recycle Bin or a permanent delete takes its entries out of the results itself (and leaves - the folders of anything else on offer), so only that request is withdrawn: folders reported - before the stop stay pending, and a whole-root request made for any other reason stays. + A move to the Recycle Bin, a permanent delete or an Undo can report so many changes that a + directory-notification feed overflows; the results show those entries already, so a whole-root + request that arrived during it is withdrawn. Folders reported meanwhile are still read again. """ - self._operation_gap = False + self._operation_gap = self._operating = False root = self.root - if self.enabled and not self._closing and root is not None and self.worker is None: + if (self.enabled and not self._closing and root is not None and self.worker is None + and self._capture is None): self.adopt(root) def request(self, path: str | None, *, whole: bool = False) -> None: @@ -324,13 +332,20 @@ def shutdown(self, *, wait: bool = True) -> None: def _merge(self, batch: ChangeBatch) -> None: if not batch.folders and not batch.trees and not batch.full: return + if batch.full and self._operating: # lost while a source operation ran: its gap, which it may withdraw + self._operation_gap = True + if not self._pending.full and not self._pending.folders and not self._pending.trees: + self._since = time.monotonic() + batch = ChangeBatch(batch.folders, trees=batch.trees) + if not batch.folders and not batch.trees: + return pending = self._pending if not pending.full and not pending.folders and not pending.trees: self._since = time.monotonic() self._pending = merged(pending, batch) def tick(self) -> None: - """Poll bounded events once a second and avoid scans during reviews, Undo offers or analysis.""" + """Poll bounded events four times a second and avoid scans during reviews, Undo offers or analysis.""" if not self.enabled or self._closing or self.window._closing: return for feed in (self._previous, self.worker): @@ -345,15 +360,29 @@ def tick(self) -> None: return elif worker.backend: if self._previous is not None: - self._retire(self._previous) - self._previous = None - if worker.since is not None and worker.backend == "usn": - self._operation_gap = False # it read on from where the stopped feed was + self._handed_over(self._previous, worker) self._set_status("follow_active", worker.backend) + if (self._stale and self.root is not None and not self.window.operation_busy + and self.window._worker is None): + self._stale = False + self.adopt(self.root) # capture again what the operation may have changed if self.window._trash_rescans: self.window._rescan_after_trash() # folders left on offer before following was turned on self._flush() + def _handed_over(self, previous: ChangeWorker, worker: ChangeWorker) -> None: + """``worker`` is armed: retire ``previous``. + + An inotify feed watches a folder only once it is armed, so the folders it watches that + ``previous`` did not are read again, which shows what changed in them meanwhile. A journal feed + reads on from the held position, and directory notifications watch the whole root anyway. + """ + self._previous = None + self._retire(previous) + if worker.backend == "inotify": + known = {folder.path for folder in previous.scope.folders} + self._merge(changed({folder.path for folder in worker.scope.folders if folder.path not in known})) + def _arm_alone(self, failed: ChangeWorker) -> None: """The new feed could not run beside the old one (an inotify watch limit): stop that one first.""" previous, root = self._previous, self.root @@ -380,6 +409,7 @@ def _flush(self) -> None: or outcome.partial or outcome.result.root is not self.root): return batch = self._pending + self._operating = False # whatever operation ran is over: a feed lost during it reads the whole root if batch.full or self._operation_gap: self._pending, self._operation_gap = ChangeBatch(), False window.rescan_folder(self.root) @@ -394,6 +424,8 @@ def _flush(self) -> None: window.refresh_folders(chosen) def _set_status(self, key: str, detail: str = "") -> None: + if (key, detail) == (self._status, self._detail) and self.label.text(): + return self._status, self._detail = key, detail self.retranslate() diff --git a/je_file_tree/gui/strings.py b/je_file_tree/gui/strings.py index 166cc57c..b3984e41 100644 --- a/je_file_tree/gui/strings.py +++ b/je_file_tree/gui/strings.py @@ -375,8 +375,8 @@ def _cjk(html: str) -> str: 'background_warning': '{detail}', 'background_low_space': 'Low available space on {root}: {free} ({percent}%)', 'action_follow_changes': 'Follow changes', - 'action_follow_changes_tip': ('Off by default. On Windows/Linux, read again only the folders that ' - 'changed, keeping their unchanged subfolders.'), + 'action_follow_changes_tip': ('Off by default. On Windows, Linux and macOS, read again only the folders ' + 'that changed, keeping their unchanged subfolders.'), 'follow_refreshed': 'Updated {count} changed folders', 'follow_refresh_failed': 'Could not update the changed folders: {reason}', 'follow_waiting': 'Follow changes: waiting for a scan', @@ -384,10 +384,11 @@ def _cjk(html: str) -> str: 'follow_active': 'Following changes ({detail})', 'follow_failed': 'Follow changes stopped: {detail}', 'follow_incomplete': 'Follow changes needs a complete physical scan', - 'follow_unsupported': 'Follow changes supports Windows and Linux', + 'follow_unsupported': 'Follow changes supports Windows, Linux and macOS', 'follow_backend_usn': 'NTFS USN', 'follow_backend_directory_changes': 'directory notifications', 'follow_backend_inotify': 'inotify', + 'follow_backend_fsevents': 'FSEvents', 'action_check_updates_tip': 'Ask PyPI over HTTPS at most once a day; notice only, with no installation.', 'update_available': 'FileTree {version} is available', 'multi_roots': 'Multiple roots', @@ -2221,7 +2222,8 @@ def _cjk(html: str) -> str: 'background_warning': '{detail}', 'background_low_space': '{root} 可用空間不足:{free}({percent}%)', 'action_follow_changes': '跟隨變更', - 'action_follow_changes_tip': '預設關閉。在 Windows/Linux 只重新讀取有變動的資料夾,保留其中未變動的子資料夾。', + 'action_follow_changes_tip': ('預設關閉。在 Windows、Linux 與 macOS 只重新讀取有變動的資料夾,' + '保留其中未變動的子資料夾。'), 'follow_refreshed': '已更新 {count} 個變動的資料夾', 'follow_refresh_failed': '無法更新變動的資料夾:{reason}', 'follow_waiting': '跟隨變更:等待掃描', @@ -2229,10 +2231,11 @@ def _cjk(html: str) -> str: 'follow_active': '跟隨變更({detail})', 'follow_failed': '跟隨變更已停止:{detail}', 'follow_incomplete': '跟隨變更需要完整的實際目錄掃描', - 'follow_unsupported': '跟隨變更支援 Windows 與 Linux', + 'follow_unsupported': '跟隨變更支援 Windows、Linux 與 macOS', 'follow_backend_usn': 'NTFS USN', 'follow_backend_directory_changes': '目錄通知', 'follow_backend_inotify': 'inotify', + 'follow_backend_fsevents': 'FSEvents', 'action_check_updates_tip': '每天最多透過 HTTPS 向 PyPI 查詢一次;僅通知,不安裝更新。', 'update_available': 'FileTree {version} 已推出', 'multi_roots': '多個掃描來源', @@ -3926,7 +3929,8 @@ def _cjk(html: str) -> str: 'background_warning': '{detail}', 'background_low_space': '{root} 可用空间不足:{free}({percent}%)', 'action_follow_changes': '跟随变更', - 'action_follow_changes_tip': '默认关闭。在 Windows/Linux 只重新读取有变化的文件夹,保留其中未变化的子文件夹。', + 'action_follow_changes_tip': ('默认关闭。在 Windows、Linux 与 macOS 只重新读取有变化的文件夹,' + '保留其中未变化的子文件夹。'), 'follow_refreshed': '已更新 {count} 个变化的文件夹', 'follow_refresh_failed': '无法更新变化的文件夹:{reason}', 'follow_waiting': '跟随变更:等待扫描', @@ -3934,10 +3938,11 @@ def _cjk(html: str) -> str: 'follow_active': '跟随变更({detail})', 'follow_failed': '跟随变更已停止:{detail}', 'follow_incomplete': '跟随变更需要完整的实际目录扫描', - 'follow_unsupported': '跟随变更支持 Windows 与 Linux', + 'follow_unsupported': '跟随变更支持 Windows、Linux 与 macOS', 'follow_backend_usn': 'NTFS USN', 'follow_backend_directory_changes': '目录通知', 'follow_backend_inotify': 'inotify', + 'follow_backend_fsevents': 'FSEvents', 'action_check_updates_tip': '每天最多通过 HTTPS 向 PyPI 查询一次;仅通知,不安装更新。', 'update_available': 'FileTree {version} 已发布', 'multi_roots': '多个扫描来源', @@ -5707,8 +5712,8 @@ def _cjk(html: str) -> str: 'background_low_space': '{root} の空き容量が少ない: {free} ({percent}%)', 'action_follow_changes': '変更を追跡する', 'action_follow_changes_tip': ( - 'デフォルトではオフです。 Windows/Linux で変更されたフォルダーだけを読み直し、変更のないサブフォルダーは' - 'そのまま残します。' + 'デフォルトではオフです。 Windows、Linux、macOS で変更されたフォルダーだけを読み直し、' + '変更のないサブフォルダーはそのまま残します。' ), 'follow_refreshed': '変更された {count} 個のフォルダーを更新しました', 'follow_refresh_failed': '変更されたフォルダーを更新できませんでした: {reason}', @@ -5717,10 +5722,11 @@ def _cjk(html: str) -> str: 'follow_active': '次の変更点 ({detail})', 'follow_failed': '変更の追跡が停止されました: {detail}', 'follow_incomplete': '変更を追跡するには完全な物理スキャンが必要です', - 'follow_unsupported': '変更を追跡すると、Windows および Linux がサポートされます', + 'follow_unsupported': '変更の追跡は Windows、Linux、macOS に対応しています', 'follow_backend_usn': 'NTFS USN', 'follow_backend_directory_changes': 'ディレクトリ通知', - 'follow_backend_inotify': '通知する', + 'follow_backend_inotify': 'inotify', + 'follow_backend_fsevents': 'FSEvents', 'action_check_updates_tip': ( '多くても 1 日に 1 回、HTTPS 経由で PyPI に問い合わせます。通知のみで、インストールはありません。' ), @@ -7810,7 +7816,8 @@ def _cjk(html: str) -> str: 'background_low_space': '{root}의 사용 가능한 공간 부족: {free} ({percent}%)', 'action_follow_changes': '변경 사항 따르기', 'action_follow_changes_tip': ( - '기본적으로 꺼져 있습니다. Windows/Linux에서 변경된 폴더만 다시 읽고, 변경되지 않은 하위 폴더는 그대로 둡니다.' + '기본적으로 꺼져 있습니다. Windows, Linux, macOS에서 변경된 폴더만 다시 읽고, ' + '변경되지 않은 하위 폴더는 그대로 둡니다.' ), 'follow_refreshed': '변경된 폴더 {count}개를 업데이트했습니다', 'follow_refresh_failed': '변경된 폴더를 업데이트할 수 없습니다: {reason}', @@ -7819,10 +7826,11 @@ def _cjk(html: str) -> str: 'follow_active': '다음 변경 사항({detail})', 'follow_failed': '변경사항 팔로우가 중지되었습니다: {detail}', 'follow_incomplete': '변경 사항을 따르려면 완전한 물리적 스캔이 필요합니다.', - 'follow_unsupported': '변경 사항에 따라 Windows 및 Linux을 지원합니다.', + 'follow_unsupported': '변경 사항 따르기는 Windows, Linux, macOS를 지원합니다', 'follow_backend_usn': 'NTFS USN', 'follow_backend_directory_changes': '디렉터리 알림', 'follow_backend_inotify': 'inotify', + 'follow_backend_fsevents': 'FSEvents', 'action_check_updates_tip': '최대 하루에 한 번 HTTPS를 통해 PyPI에 문의하세요. 설치 없이 참고만 가능합니다.', 'update_available': 'FileTree {version} 사용 가능', 'multi_roots': '여러 뿌리', diff --git a/progress.md b/progress.md index 4a8db8d8..fb99e399 100644 --- a/progress.md +++ b/progress.md @@ -23,7 +23,7 @@ ### 掃描 - **#142**(第二優先)[待驗證] 由主檔案表讀取 NTFS 磁碟的掃描(U-20261115-05、-07)只在一台機器的一顆系統磁碟上量測與比對。尚未驗證:檔案紀錄為 4096 位元組的磁碟(4Kn);檔案表本身高度分散的磁碟;資料磁碟與外接磁碟;啟用系統壓縮(CompactOS)或含大量雲端預留位置的磁碟;真正的 Windows 10;冷快取下與列出資料夾的秒數比較;CI 的 Windows 工作能否讀取檔案表(讀不到時 `test_mft_scan.py` 的實機測試會略過);已發行的 wheel 與獨立版是否含新版模組;有日光節約時間的時區下原生時間文字與 Python 是否相同(U-20261115-10)。另有 85 個資料夾的大小與 Windows 回報不同,原因未查明;這些項目在操作前會被判定為已變更而拒絕,重新掃描該資料夾即可。 -- **#144**(第二優先)[待驗證] 「跟隨變更」只重新讀取變動資料夾(U-20261120-01),Windows 的 NTFS USN 與目錄通知已在本機原生驗證(U-20261120-02)。尚未驗證:Linux inotify 新舊監看短暫並存時,是否會超過監看數量上限;在使用者實際使用、變動頻繁的資料夾上,重新讀取監看範圍的背景負擔。 +- **#144**(第二優先)[待驗證] 「跟隨變更」只重新讀取變動資料夾(U-20261120-01),Windows 的 NTFS USN 與目錄通知已在本機原生驗證(U-20261120-02)。macOS FSEvents 監看(U-20261120-03)只在 CI 的 macOS 工作上原生執行。尚未驗證:在實體 Mac 上的 Finder 操作與外接磁碟;Linux inotify 新舊監看短暫並存時,是否會超過監看數量上限;在使用者實際使用、變動頻繁的資料夾上,重新讀取監看範圍的背景負擔。 - **#138**(第三優先)[待驗證] 選用的原生掃描模組 `je_file_tree._native` 已隨 0.2.9 發行(PyPI 的 Windows wheel 與 Windows 獨立版),發行本身已驗證。尚未驗證的環境:真正的 Windows 10;未安裝 Visual C++ 可轉散發套件的乾淨機器(模組須可載入並被採用);非 NTFS 磁碟區中的 exFAT 與 ReFS(FAT32 已驗證),須確認掃描起點比對會正確決定是否使用原生讀取;ARM64 的 Windows(目前只發行 x64 wheel)。目前沒有這些環境。任何一項失敗時,程式仍以 Python 讀取掃描,結果相同。 - **#35**(第二優先)[受阻] 須在實際緩慢的遠端 SMB 連線上,量測 UNC 路徑與對應磁碟機在不同背景工作數量下的處理量,再建議該環境的並行程度。使用者目前未提供遠端共用資料夾或慢速連線驗證環境,本機回送連線的原生量測無法證明遠端效能。 - **#75**(第一優先)[受阻] 須透過原生目錄走訪防護,驗證 macOS 與其他 POSIX 系統上同一裝置內的掛載邊界。使用者目前未提供可拋棄的對應驗證環境。FreeBSD 的 nullfs 具有不同的 `st_dev`,無法證明此情境。未掛載的 APFS 同容器磁碟區容量仍未知。圖形介面與命令列的容量帳目仍為估計值;無法獨立量測的共用空間、保留空間、檔案系統中繼資料、遺漏位元組及其他磁碟區總量,均須維持未知。直接顯示垃圾桶標籤與 Finder 清空功能仍由 #21 追蹤。 diff --git a/test/test_change_watch.py b/test/test_change_watch.py index f7a697c5..b09fd2cf 100644 --- a/test/test_change_watch.py +++ b/test/test_change_watch.py @@ -10,7 +10,7 @@ import pytest from je_file_tree.core import change_watch as change -from je_file_tree.core import linux_watch, windows_watch +from je_file_tree.core import linux_watch, mac_watch, windows_watch from je_file_tree.core.scanner import scan @@ -184,8 +184,61 @@ def test_usn_uses_captured_ids_and_external_shared_alias_changes_refresh_known_p assert not outside.folders and not outside.full _, shared = windows_watch.parse_usn(_usn(parent=99), {22: root}, root, 100, {55: (root,)}) assert shared.folders == (root,) - _, aliases = windows_watch.parse_usn(_usn(reason=0x10000), {22: root}, root, 100) - assert aliases.full and aliases.reason == "hard_links_changed" + _, inside = windows_watch.parse_usn(_usn(reason=0x10000), {22: root}, root, 100) + assert inside.folders == (root,) and not inside.full # a name made inside: its folder + _, aliases = windows_watch.parse_usn(_usn(reason=0x10000, parent=99), {22: root}, root, 100) + assert aliases.full and aliases.reason == "hard_links_changed" # elsewhere, and no way to look it up + linked = set() + _, looked_up = windows_watch.parse_usn(_usn(reason=0x10000, parent=99), {22: root}, root, 100, None, linked) + assert linked == {55} and not looked_up.full and not looked_up.folders + + +def test_a_file_relinked_elsewhere_reads_only_the_folders_holding_its_names(monkeypatch): + names = {1: ["\\owned\\資料\\file", "\\elsewhere\\alias"], 2: ["\\elsewhere\\only"]} + + def names_of(_kernel, _volume, number): + if number == 3: + raise OSError(None, "gone", None, 2) + if number == 4: + raise OSError(None, "denied", None, 5) + return names[number] + + monkeypatch.setattr(windows_watch, "_names_of", names_of) + assert windows_watch._linked_folders(None, None, {1, 2, 3}, "C:\\", "C:\\owned") == {"C:\\owned\\資料"} + assert windows_watch._linked_folders(None, None, {1, 4}, "C:\\", "C:\\owned") is None + assert windows_watch._linked_folders(None, None, set(range(100)), "C:\\", "C:\\owned") is None + + +def test_a_busy_journal_is_read_on_at_once_and_notification_buffers_fit_the_drive(monkeypatch): + folder = change.WatchFolder("C:\\owned", 7, 22) + kernel = SimpleNamespace(CreateFileW=lambda *_args: 42, CloseHandle=lambda _handle: None) + reads, waits = [], [] + + def control(_kernel, _handle, code, data=b""): + if code == windows_watch._QUERY_USN: + return windows_watch._JOURNAL.pack(9, 50, 400, 50, 0, 4096, 1024) + start = windows_watch._READ.unpack(data)[0] + reads.append(start) + if len(reads) == 3: + cancel.set() + return struct.pack(" Date: Fri, 20 Nov 2026 15:55:45 +0800 Subject: [PATCH 4/4] Steady two timing-sensitive tests and a Windows-only lookup test The elevation tests released their held launch after 2 s as a fallback; on a loaded runner that came first, so the launch had ended before the close request looked. The recurring review waited 10 s for a scan, analysis, review and move in a row. Both wait longer now. The name lookup test simulated a gone file with OSError's winerror argument, which only Windows sets; it sets the attribute itself now. --- docs/updates/2026-11.md | 18 ++++++++++++++++++ docs/updates/README.md | 1 + test/test_change_watch.py | 9 +++++++-- test/test_elevation_worker.py | 12 ++++++++---- test/test_recurring_review.py | 4 +++- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/updates/2026-11.md b/docs/updates/2026-11.md index b50c7bd3..dca7fa26 100644 --- a/docs/updates/2026-11.md +++ b/docs/updates/2026-11.md @@ -783,3 +783,21 @@ - **未驗證**:macOS FSEvents 只能在 CI 的 macOS 工作上原生執行,本機沒有 Mac;Linux inotify 交接只由 CI 涵蓋。 - **檔案**:`je_file_tree/core/mac_watch.py`(新)、`core/change_watch.py`、`core/windows_watch.py`;`je_file_tree/gui/change_watch.py`、`gui/strings.py`;`architecture.md`、`docs/core-api.md`、`docs/guide.md` 與四個翻譯;`progress.md`;測試檔案。 - **待辦**:`progress.md` #144。 + +## U-20261120-04 · 2026-11-20 · 修正兩個偶爾失敗的測試,以及只在 Windows 能通過的名稱查詢測試 · #test #ci + +- **原因**:使用者要求處理剩餘問題,其中包括兩個偶爾失敗的測試。另外 U-20261120-03 推送後,CI 的 Linux 與 macOS 工作各失敗一個新測試。 +- **`test_elevation_worker.py`**(2026-11-19 在 Windows / Python 3.11 失敗一次): + - 失敗內容:觀察到的是 `(False, False, False)`,也就是檢查時權限提升的工作已經結束。 + - 原因:防止測試卡死的備援計時器在 2 秒後放行;負載重的 CI 在那之前還沒輪到檢查,工作先結束了。 + - 修正:同一檔案中兩個測試的備援放行與等待都改為 10 秒。正常情況下,檢查後 50 ms 就會放行,所以測試不會變慢。 +- **`test_recurring_review.py`**(2026-11-19 在 macOS 逾時一次): + - 原因:這個測試依序跑重新掃描、分析、清理建議、審查與移到資源回收筒;本機 0.6 秒,在負載重的 macOS CI 上曾超過 10 秒。 + - 修正:這次等待改為 30 秒。 +- **`test_change_watch.py` 的名稱查詢測試**(U-20261120-03 新增):以 `OSError(None, ..., None, 2)` 模擬已刪除的檔案,但 `winerror` 只有 Windows 會設定,所以在 Linux 與 macOS 失敗。改為明確設定 `winerror`。 +- **同一次 CI 的原生結果**: + - macOS 上的 FSEvents 原生測試通過:核心監看測試、視窗中只更新一個資料夾的測試。 + - Windows 上以 NTFS 變更日誌執行的硬連結原生測試通過。 +- **驗證**:`test_elevation_worker.py`、`test_recurring_review.py` 共 19 個測試通過;名稱查詢測試通過;`ruff check .` 通過。 +- **檔案**:`test/test_elevation_worker.py`、`test/test_recurring_review.py`、`test/test_change_watch.py`。 +- **待辦**:無。 diff --git a/docs/updates/README.md b/docs/updates/README.md index 5cd7cd30..a92bb483 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -70,6 +70,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261120-04 | 2026-11-20 | 修正兩個偶爾失敗的測試,以及只在 Windows 能通過的名稱查詢測試 | #test #ci | [2026-11](2026-11.md) | | U-20261120-03 | 2026-11-20 | 「跟隨變更」在操作期間不中斷、硬連結只讀相關資料夾,並支援 macOS | #feature #perf #gui #macos | [2026-11](2026-11.md) | | U-20261120-02 | 2026-11-20 | 在本機原生驗證「跟隨變更」只讀變動資料夾,並修正兩處多餘工作 | #validation #perf #gui | [2026-11](2026-11.md) | | U-20261120-01 | 2026-11-20 | 「跟隨變更」只重新讀取變動的資料夾,不再重掃整個掃描 | #perf #gui #scan | [2026-11](2026-11.md) | diff --git a/test/test_change_watch.py b/test/test_change_watch.py index b09fd2cf..89687abd 100644 --- a/test/test_change_watch.py +++ b/test/test_change_watch.py @@ -196,11 +196,16 @@ def test_usn_uses_captured_ids_and_external_shared_alias_changes_refresh_known_p def test_a_file_relinked_elsewhere_reads_only_the_folders_holding_its_names(monkeypatch): names = {1: ["\\owned\\資料\\file", "\\elsewhere\\alias"], 2: ["\\elsewhere\\only"]} + def failure(code): + error = OSError("owned lookup failure") + error.winerror = code # what ctypes.WinError sets; plain OSError has none outside Windows + return error + def names_of(_kernel, _volume, number): if number == 3: - raise OSError(None, "gone", None, 2) + raise failure(2) # the file is gone if number == 4: - raise OSError(None, "denied", None, 5) + raise failure(5) # access denied return names[number] monkeypatch.setattr(windows_watch, "_names_of", names_of) diff --git a/test/test_elevation_worker.py b/test/test_elevation_worker.py index 75247462..baba6852 100644 --- a/test/test_elevation_worker.py +++ b/test/test_elevation_worker.py @@ -51,7 +51,7 @@ def test_close_waits_for_launch_and_success_still_closes_the_original(window, qa def relaunch(arguments): calls.append(list(arguments)) entered.set() - release.wait(3) + release.wait(10) return started monkeypatch.setattr(elevation, "relaunch_elevated", relaunch) @@ -72,7 +72,9 @@ def close_request(): timer.timeout.connect(close_request) timer.start() - fallback = threading.Timer(2, release.set) + # Only so that nothing hangs if the request never runs. At 2 s it once fired first on a loaded Windows + # CI runner (2026-11-19): the launch ended before the request could observe it still running. + fallback = threading.Timer(10, release.set) fallback.start() try: window.restart_as_admin() @@ -119,7 +121,7 @@ def test_parent_close_waits_for_native_call_without_late_status(window, qapp, mo def relaunch(_arguments): entered.set() - release.wait(3) + release.wait(10) return False monkeypatch.setattr(elevation, "relaunch_elevated", relaunch) @@ -134,7 +136,9 @@ def close_parent(): timer.timeout.connect(close_parent) timer.start() - fallback = threading.Timer(2, release.set) + # Only so that nothing hangs if the request never runs. At 2 s it once fired first on a loaded Windows + # CI runner (2026-11-19): the launch ended before the request could observe it still running. + fallback = threading.Timer(10, release.set) fallback.start() try: window.restart_as_admin() diff --git a/test/test_recurring_review.py b/test/test_recurring_review.py index 8e3704d4..5cdc34ef 100644 --- a/test/test_recurring_review.py +++ b/test/test_recurring_review.py @@ -154,7 +154,9 @@ def run(worker): monkeypatch.setattr(trash_worker.TrashWorker, "run", run) owner = monitored.add_tab() RecurringFlow(monitor, owner, current).start() - _wait(qapp, lambda: bool(workers) and owner._trash_worker is None) + # A fresh scan, its analysis, clean-up suggestions, the review and the move run one after another: + # 0.6 s here, but past 10 s once on a loaded macOS CI runner (2026-11-19), so this waits longer. + _wait(qapp, lambda: bool(workers) and owner._trash_worker is None, timeout=30) assert reviewed and not moved and workers[0].result.skipped[0][1] == "proposal_changed" assert (source / "one.dmp").read_bytes() == b"owned one" and (source / "two.dmp").read_bytes() == b"owned two"