Description of problem: ------------------------ Synchronization stops after a few minutes when syncing a large volume of small file data. Version-Release number of selected component (if applicable): ------------- ceph version 18.2.1-229.el9cp (ef652b206f2487adfc86613646a4cac946f6b4e0) reef (stable) How reproducible: ----------------- 2/2 Steps to Reproduce: 1. Create a subvolume 2. Create large volume of smallfile data : # python3 /home/cephuser/smallfile/smallfile_cli.py --operation create --threads 8 --file-size 1024 --files 10000 --dirs-per-dir 200 --files-per-dir 200 --top /mnt/subv2/smallfile_io_1/ 3. Enable cephfs mirroring and add the subvolume path for data synchronizing. 4. Keep monitoring the sync status until the sync completes. Actual results: Sync stopped after few mins. Expected results: Sync should complete the transfer ======= [root@extensa022 34a69174-0cae-11ef-b250-b49691cee884]# ceph --admin-daemon ceph-client.cephfs-mirror.extensa022.yirwnl.2.93866436942112.asok fs mirror peer status cephfs@9 8f483499-58c7-47ef-bee6-530e1408b7c2 { "/volumes/_nogroup/subv2": { "state": "syncing", "current_sycning_snap": { "id": 10, "name": "snap1" }, "snaps_synced": 0, "snaps_deleted": 0, "snaps_renamed": 0 } } [root@extensa022 34a69174-0cae-11ef-b250-b49691cee884]# Sync status doesn't show anything else apart from this, even though is no increase in data size on the remote cluster. From the logs, I don't see any errors regarding why the sync is not proceeding. ====== 2024-08-08T15:01:11.476+0000 7f88393a1640 20 cephfs::mirror::InstanceWatcher handle_notify 2024-08-08T15:01:11.476+0000 7f88393a1640 20 cephfs::mirror::InstanceWatcher handle_notify: notifier_id=3095229, dir_path=/volumes/_nogroup/subv2, mode=acquire 2024-08-08T15:01:11.476+0000 7f88393a1640 5 cephfs::mirror::FSMirror handle_acquire_directory: dir_path=/volumes/_nogroup/subv2 2024-08-08T15:01:11.476+0000 7f88393a1640 10 cephfs::mirror::ServiceDaemon: 0x555efb996820 add_or_update_fs_attribute: fscid=9 2024-08-08T15:01:11.476+0000 7f88393a1640 10 cephfs::mirror::ServiceDaemon: 0x555efb996820 schedule_update_status 2024-08-08T15:01:11.476+0000 7f88393a1640 10 cephfs::mirror::FSMirror handle_acquire_directory: peer={uuid=8f483499-58c7-47ef-bee6-530e1408b7c2, remote_cluster={client_name=client.mirror_remote, cluster_name=remote, fs_name=cephfs}} 2024-08-08T15:01:11.476+0000 7f88393a1640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) add_directory: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:01:11.886+0000 7f8839ba2640 20 cephfs::mirror::Watcher handle_notify: notify_id=18283675801097, handle=93866452824576, notifier_id=3095229 2024-08-08T15:01:11.886+0000 7f8839ba2640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T15:01:12.183+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: trying to pick from 1 directories 2024-08-08T15:01:12.183+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pick_directory 2024-08-08T15:01:12.183+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: picked dir_root=/volumes/_nogroup/subv2 2024-08-08T15:01:12.183+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) register_directory: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:01:12.183+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) try_lock_directory: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:01:12.222+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) try_lock_directory: dir_root=/volumes/_nogroup/subv2 locked 2024-08-08T15:01:12.222+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) register_directory: dir_root=/volumes/_nogroup/subv2 registered with replayer=0x555efcade6c0 2024-08-08T15:01:12.223+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) sync_snaps: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:01:12.223+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_sync_snaps: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:01:12.223+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: dir_root=/volumes/_nogroup/subv2, snap_dir=/volumes/_nogroup/subv2/.snap, is_remote=0 . . . 2024-08-08T15:02:12.237+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: trying to pick from 1 directories 2024-08-08T15:02:12.237+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pick_directory 2024-08-08T15:02:12.237+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: picked dir_root=/volumes/_nogroup/subv2 2024-08-08T15:02:12.237+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) register_directory: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:02:12.237+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) try_lock_directory: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:02:12.238+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) try_lock_directory: dir_root=/volumes/_nogroup/subv2 locked 2024-08-08T15:02:12.238+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) register_directory: dir_root=/volumes/_nogroup/subv2 registered with replayer=0x555efcade6c0 2024-08-08T15:02:12.238+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) sync_snaps: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:02:12.238+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_sync_snaps: dir_root=/volumes/_nogroup/subv2 2024-08-08T15:02:12.238+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: dir_root=/volumes/_nogroup/subv2, snap_dir=/volumes/_nogroup/subv2/.snap, is_remote=0 2024-08-08T15:02:12.238+0000 7f8824b78640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: trying to pick from 1 directories 2024-08-08T15:02:12.238+0000 7f8824b78640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pick_directory 2024-08-08T15:02:12.239+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: entry=. 2024-08-08T15:02:12.239+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: entry=.. 2024-08-08T15:02:12.239+0000 7f8824377640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: trying to pick from 1 directories 2024-08-08T15:02:12.239+0000 7f8824377640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pick_directory 2024-08-08T15:02:12.239+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: entry=snap1 2024-08-08T15:02:12.240+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: local snap_map={10=snap1} 2024-08-08T15:02:12.240+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: dir_root=/volumes/_nogroup/subv2, snap_dir=/volumes/_nogroup/subv2/.snap, is_remote=1 2024-08-08T15:02:12.240+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: entry=. 2024-08-08T15:02:12.241+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: entry=.. 2024-08-08T15:02:12.241+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) build_snap_map: remote snap_map={} 2024-08-08T15:02:12.241+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_snap_deletes: dir_root=/volumes/_nogroup/subv2, deleted snapshots= 2024-08-08T15:02:12.241+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_snap_renames: dir_root=/volumes/_nogroup/subv2, renamed snapshots= 2024-08-08T15:02:12.241+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_sync_snaps: last snap-id transferred=0 2024-08-08T15:02:12.241+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_sync_snaps: synchronizing from snap-id=10 2024-08-08T15:02:12.241+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) synchronize: dir_root=/volumes/_nogroup/subv2, current=snap1,10 2024-08-08T15:02:12.241+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) synchronize: missing "ceph.mirror.dirty_snap_id" xattr on remote -- using incremental sync with remote scan . . . 2024-08-08T15:02:12.241+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: dir_root=/volumes/_nogroup/subv2, current=snap1,10 2024-08-08T15:02:12.241+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pre_sync_check_and_open_handles: dir_root=/volumes/_nogroup/subv2, current=snap1,10 2024-08-08T15:02:12.241+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) open_dir: dir_path=/volumes/_nogroup/subv2/.snap/snap1 2024-08-08T15:02:12.241+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) open_dir: expected snapshot id=10 2024-08-08T15:02:12.242+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) open_dir: dir_path=/volumes/_nogroup/subv2 2024-08-08T15:02:12.242+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pre_sync_check_and_open_handles: using remote dir_root for incremental transfer 2024-08-08T15:02:12.243+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: 1 entries in stack 2024-08-08T15:02:12.243+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: top of stack path=. 2024-08-08T15:02:12.243+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: dir_root=/volumes/_nogroup/subv2, epath=. 2024-08-08T15:02:12.244+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: reached EOD 2024-08-08T15:02:12.245+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) remote_mkdir: remote epath=./c9f07692-11c9-4198-9470-5926b874a382 2024-08-08T15:02:12.246+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: 2 entries in stack 2024-08-08T15:02:12.246+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: top of stack path=./c9f07692-11c9-4198-9470-5926b874a382 2024-08-08T15:02:12.246+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382 2024-08-08T15:02:12.248+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: reached EOD 2024-08-08T15:02:12.249+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) remote_mkdir: remote epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1 2024-08-08T15:02:12.280+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: 3 entries in stack 2024-08-08T15:02:12.280+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: top of stack path=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1 2024-08-08T15:02:12.280+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1 2024-08-08T15:02:12.282+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: reached EOD 2024-08-08T15:02:12.283+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) remote_mkdir: remote epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_dstdir 2024-08-08T15:02:12.313+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: 4 entries in stack 2024-08-08T15:02:12.313+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: top of stack path=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_dstdir 2024-08-08T15:02:12.313+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_dstdir 2024-08-08T15:02:12.315+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: reached EOD 2024-08-08T15:02:12.316+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) remote_mkdir: remote epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_dstdir/magna030 2024-08-08T15:02:12.346+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: 5 entries in stack 2024-08-08T15:02:12.346+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: top of stack path=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_dstdir/magna030 2024-08-08T15:02:12.346+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_dstdir/magna030 2024-08-08T15:02:12.348+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) propagate_deleted_entries: reached EOD . . . 2024-08-08T15:04:01.316+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) remote_file_op: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7549_, need_data_sync=1, need_attr_sync=1 2024-08-08T15:04:01.316+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) copy_to_remote: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7549_ 2024-08-08T15:04:01.382+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: done for epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7549_ 2024-08-08T15:04:01.382+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: 7 entries in stack 2024-08-08T15:04:01.382+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: top of stack path=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037 2024-08-08T15:04:01.382+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: 8 entries in stack 2024-08-08T15:04:01.382+0000 7f8825379640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: top of stack path=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7423_ 2024-08-08T15:04:01.382+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) should_sync_entry: epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7423_ 2024-08-08T15:04:01.382+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) should_sync_entry: entry=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7423_, r=-2 2024-08-08T15:04:01.382+0000 7f8825379640 5 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) do_synchronize: entry=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7423_, data_sync=1, attr_sync=1 2024-08-08T15:04:01.382+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) remote_file_op: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7423_, need_data_sync=1, need_attr_sync=1 2024-08-08T15:04:01.382+0000 7f8825379640 10 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) copy_to_remote: dir_root=/volumes/_nogroup/subv2, epath=./c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/file_srcdir/magna030/thrd_01/d_037/_magna030_01_7423_ 2024-08-08T15:04:02.060+0000 7f8839ba2640 20 cephfs::mirror::Watcher handle_notify: notify_id=18287970768563, handle=93866452824576, notifier_id=3095229 2024-08-08T15:04:02.060+0000 7f8839ba2640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T15:04:02.251+0000 7f8824377640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: trying to pick from 1 directories 2024-08-08T15:04:02.251+0000 7f8824377640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pick_directory 2024-08-08T15:04:03.061+0000 7f88393a1640 20 cephfs::mirror::Watcher handle_notify: notify_id=18287970768564, handle=93866452824576, notifier_id=3095229 2024-08-08T15:04:03.061+0000 7f88393a1640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T15:04:03.192+0000 7f88423b3640 20 cephfs::mirror::Mirror update_fs_mirrors 2024-08-08T15:04:03.192+0000 7f88423b3640 20 cephfs::mirror::Mirror schedule_mirror_update_task: scheduling fs mirror update (0x555efc715d60) after 2 seconds . . . . 2024-08-08T19:33:09.426+0000 7f88393a1640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:10.409+0000 7f88423b3640 20 cephfs::mirror::Mirror update_fs_mirrors 2024-08-08T19:33:10.409+0000 7f88423b3640 20 cephfs::mirror::Mirror schedule_mirror_update_task: scheduling fs mirror update (0x555efcae24a0) after 2 seconds 2024-08-08T19:33:10.427+0000 7f8839ba2640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457655, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:10.427+0000 7f8839ba2640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:11.428+0000 7f88393a1640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457656, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:11.428+0000 7f88393a1640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:12.410+0000 7f88423b3640 20 cephfs::mirror::Mirror update_fs_mirrors 2024-08-08T19:33:12.410+0000 7f88423b3640 20 cephfs::mirror::Mirror schedule_mirror_update_task: scheduling fs mirror update (0x555efc715d60) after 2 seconds 2024-08-08T19:33:12.429+0000 7f8839ba2640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457657, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:12.429+0000 7f8839ba2640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:13.429+0000 7f88393a1640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457658, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:13.429+0000 7f88393a1640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:14.410+0000 7f88423b3640 20 cephfs::mirror::Mirror update_fs_mirrors 2024-08-08T19:33:14.410+0000 7f88423b3640 20 cephfs::mirror::Mirror schedule_mirror_update_task: scheduling fs mirror update (0x555efcae24a0) after 2 seconds 2024-08-08T19:33:14.430+0000 7f8839ba2640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457659, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:14.430+0000 7f8839ba2640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:15.431+0000 7f88393a1640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457660, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:15.431+0000 7f88393a1640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:16.183+0000 7f8824b78640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: trying to pick from 1 directories 2024-08-08T19:33:16.183+0000 7f8824b78640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pick_directory 2024-08-08T19:33:16.183+0000 7f8824377640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) run: trying to pick from 1 directories 2024-08-08T19:33:16.183+0000 7f8824377640 20 cephfs::mirror::PeerReplayer(8f483499-58c7-47ef-bee6-530e1408b7c2) pick_directory 2024-08-08T19:33:16.410+0000 7f88423b3640 20 cephfs::mirror::Mirror update_fs_mirrors 2024-08-08T19:33:16.410+0000 7f88423b3640 20 cephfs::mirror::Mirror schedule_mirror_update_task: scheduling fs mirror update (0x555efc715d60) after 2 seconds 2024-08-08T19:33:16.432+0000 7f8839ba2640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457661, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:16.432+0000 7f8839ba2640 20 cephfs::mirror::MirrorWatcher handle_notify 2024-08-08T19:33:17.433+0000 7f88393a1640 20 cephfs::mirror::Watcher handle_notify: notify_id=18330920457662, handle=93866452824576, notifier_id=3095229 2024-08-08T19:33:17.433+0000 7f88393a1640 20 cephfs::mirror::MirrorWatcher handle_notify ===== The snapshot was created at: 2024-08-08T15:02:12.239+0000 and even after 4 hours the sync status remains as syncing but the transferred data to remote shows as only 1.2GB out of 79GB. ===== [root@magna030 ~]# ceph fs subvolume snapshot info cephfs subv2 snap1 { "created_at": "2024-08-08 15:02:10.011772", "data_pool": "cephfs.cephfs.data", "has_pending_clones": "no" } [root@magna030 ~]# ceph fs subvolume info cephfs subv2 { "atime": "2024-08-08 08:28:44", "bytes_pcent": "undefined", "bytes_quota": "infinite", "bytes_used": 83886080010, "created_at": "2024-08-08 08:28:44", "ctime": "2024-08-08 08:31:31", "data_pool": "cephfs.cephfs.data", "features": [ "snapshot-clone", "snapshot-autoprotect", "snapshot-retention" ], "flavor": 2, "gid": 0, "mode": 16877, "mon_addrs": [ "10.8.130.25:6789", "10.8.130.27:6789", "10.8.130.26:6789" ], "mtime": "2024-08-08 08:31:31", "path": "/volumes/_nogroup/subv2/c9f07692-11c9-4198-9470-5926b874a382", "pool_namespace": "", "state": "complete", "type": "subvolume", "uid": 0 } [root@magna030 smallfile_io_1]# cd /mnt/subv2/smallfile_io_1/ [root@magna030 smallfile_io_1]# du -h ../smallfile_io_1/ 79G ../smallfile_io_1/file_srcdir/magna030 79G ../smallfile_io_1/file_srcdir 79G ../smallfile_io_1/ == On Remote : [root@magna029 c9f07692-11c9-4198-9470-5926b874a382]# cd /mnt/subv/volumes/_nogroup/subv2/c9f07692-11c9-4198-9470-5926b874a382/smallfile_io_1/ [root@magna029 smallfile_io_1]# du -h * 1.2G smallfile_io_1/file_srcdir/magna030 1.2G smallfile_io_1/file_srcdir 1.2G smallfile_io_1