Description of problem: ----------------------- After destroying the cephfs mirroring configuration, the metrics still reports the mirrored_filesystem. [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs snapshot mirror daemon status Error ENOTSUP: Module 'mirroring' is not enabled/loaded (required by command 'fs snapshot mirror daemon status'): use `ceph mgr module enable mirroring` to enable it ========= Executed the automated Tests to Configure and Destroy the Mirroring - http://magna002.ceph.redhat.com/cephci-jenkins/cephci-run-H6MTU0/ [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs ls name: cephfs, metadata pool: cephfs.cephfs.meta, data pools: [cephfs.cephfs.data ] [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph mgr module enable mirroring [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs snapshot mirror disable cephfs {} [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs snapshot mirror peer_list Error EINVAL: filesystem None is not mirrored [root@ceph1-hk-m-zy7rnm-node6 7abd8508-e505-11ee-bbf4-fa163e6ab61d]# ceph --admin-daemon ceph-client.cephfs-mirror.ceph1-hk-m-zy7rnm-node6.yvtajb.2.94028663044328.asok help { "config diff": "dump diff of current config and default config", "config diff get": "dump diff get <field>: dump diff of current and default config setting <field>", "config get": "config get <field>: get the config value", "config help": "get config setting schema and descriptions", "config set": "config set <field> <val> [<val> ...]: set a config variable", "config show": "dump current config settings", "config unset": "config unset <field>: unset a config variable", "counter dump": "dump all labeled and non-labeled counters and their values", "counter schema": "dump all labeled and non-labeled counters schemas", "dump_mempools": "get mempool stats", "get_command_descriptions": "list available commands", "git_version": "get git sha1", "help": "list available commands", "injectargs": "inject configuration arguments into running daemon", "log dump": "dump recent log entries to log file", "log flush": "flush log entries to log file", "log reopen": "reopen log file", "objecter_requests": "show in-progress osd requests", "perf dump": "dump non-labeled counters and their values", "perf histogram dump": "dump perf histogram values", "perf histogram schema": "dump perf histogram schema", "perf reset": "perf reset <name>: perf reset all or one perfcounter name", "perf schema": "dump non-labeled counters schemas", "rotate-key": "rotate live authentication key", "version": "get ceph version" } [root@ceph1-hk-m-zy7rnm-node6 7abd8508-e505-11ee-bbf4-fa163e6ab61d]# [root@ceph1-hk-m-zy7rnm-node6 7abd8508-e505-11ee-bbf4-fa163e6ab61d]# ceph --admin-daemon ceph-client.cephfs-mirror.ceph1-hk-m-zy7rnm-node6.yvtajb.2.94028663044328.asok counter dump | jq -r '. | to_entries | map(select(.key | match("cephfs_mirror")))' [ { "key": "cephfs_mirror", "value": [ { "labels": {}, "counters": { "mirrored_filesystems": 1, "mirror_enable_failures": 0 } } ] } ]
(In reply to Hemanth Kumar from comment #0) > Description of problem: > ----------------------- > After destroying the cephfs mirroring configuration, the metrics still > reports the mirrored_filesystem. > > [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs snapshot mirror daemon > status > Error ENOTSUP: Module 'mirroring' is not enabled/loaded (required by command > 'fs snapshot mirror daemon status'): use `ceph mgr module enable mirroring` > to enable it > > > ========= > > Executed the automated Tests to Configure and Destroy the Mirroring - > http://magna002.ceph.redhat.com/cephci-jenkins/cephci-run-H6MTU0/ > > [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs ls > name: cephfs, metadata pool: cephfs.cephfs.meta, data pools: > [cephfs.cephfs.data ] > > [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph mgr module enable > mirroring > > > [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs snapshot mirror > disable cephfs > {} > [root@ceph1-hk-m-zy7rnm-node8 subvolgroup_1]# ceph fs snapshot mirror > peer_list > Error EINVAL: filesystem None is not mirrored > > [root@ceph1-hk-m-zy7rnm-node6 7abd8508-e505-11ee-bbf4-fa163e6ab61d]# ceph > --admin-daemon > ceph-client.cephfs-mirror.ceph1-hk-m-zy7rnm-node6.yvtajb.2.94028663044328. > asok help > { > "config diff": "dump diff of current config and default config", > "config diff get": "dump diff get <field>: dump diff of current and > default config setting <field>", > "config get": "config get <field>: get the config value", > "config help": "get config setting schema and descriptions", > "config set": "config set <field> <val> [<val> ...]: set a config > variable", > "config show": "dump current config settings", > "config unset": "config unset <field>: unset a config variable", > "counter dump": "dump all labeled and non-labeled counters and their > values", > "counter schema": "dump all labeled and non-labeled counters schemas", > "dump_mempools": "get mempool stats", > "get_command_descriptions": "list available commands", > "git_version": "get git sha1", > "help": "list available commands", > "injectargs": "inject configuration arguments into running daemon", > "log dump": "dump recent log entries to log file", > "log flush": "flush log entries to log file", > "log reopen": "reopen log file", > "objecter_requests": "show in-progress osd requests", > "perf dump": "dump non-labeled counters and their values", > "perf histogram dump": "dump perf histogram values", > "perf histogram schema": "dump perf histogram schema", > "perf reset": "perf reset <name>: perf reset all or one perfcounter > name", > "perf schema": "dump non-labeled counters schemas", > "rotate-key": "rotate live authentication key", > "version": "get ceph version" > } > [root@ceph1-hk-m-zy7rnm-node6 7abd8508-e505-11ee-bbf4-fa163e6ab61d]# > > [root@ceph1-hk-m-zy7rnm-node6 7abd8508-e505-11ee-bbf4-fa163e6ab61d]# ceph > --admin-daemon > ceph-client.cephfs-mirror.ceph1-hk-m-zy7rnm-node6.yvtajb.2.94028663044328. > asok counter dump | jq -r '. | to_entries | map(select(.key | > match("cephfs_mirror")))' > [ > { > "key": "cephfs_mirror", > "value": [ > { > "labels": {}, > "counters": { > "mirrored_filesystems": 1, > "mirror_enable_failures": 0 > } > } > ] > } > ] This should be a bug. Jos, please create a tracker and link it in this BZ.
@Hemanth This doesn't exist in the upstream. We've backported all metrics patches downstream, as discussed with Venky. After disabling, wait for sometime and issue the 'counter dump' command and check if 'mirrored_filesystems' shows correct value (asynchronous)?
Btw, your peer_list command is wrong.