Description of problem: Devices file PVID <ID> last seen on /dev/<device> not found after wipefs Version-Release number of selected component (if applicable): kernel-5.14.0-63.el9 device-mapper-1.02.183-4.el9 lvm2-2.03.14-4.el9 Steps to Reproduce: [root@localhost ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command pvcreate pid 4902 at Tue Apr 5 14:57:49 2022 VERSION=1.1.79 IDTYPE=devname IDNAME=/dev/sdf2 DEVNAME=/dev/sdf2 PVID=yNatmOaFZE0L2UIOv5lmhwHfqtAwYfhq PART=2 IDTYPE=devname IDNAME=/dev/sdd DEVNAME=/dev/sdd PVID=. IDTYPE=devname IDNAME=/dev/sde DEVNAME=/dev/sde PVID=. IDTYPE=devname IDNAME=/dev/sdb DEVNAME=/dev/sdb PVID=. IDTYPE=devname IDNAME=/dev/sdc DEVNAME=/dev/sdc PVID=. IDTYPE=devname IDNAME=/dev/sda DEVNAME=/dev/sda PVID=. [root@localhost ~]# pvcreate /dev/sda Physical volume "/dev/sda" successfully created. [root@localhost ~]# wipefs -a /dev/sda /dev/sda: 8 bytes were erased at offset 0x00000218 (LVM2_member): 4c 56 4d 32 20 30 30 31 [root@localhost ~]# pvs Devices file PVID Sio0VbF2Of6IVA4DVIe8ycUFXv1MzHz1 not found on device /dev/sda. PV VG Fmt Attr PSize PFree /dev/sdf2 rhel lvm2 a-- <9.00g 0 [root@localhost ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command pvs pid 4949 at Tue Apr 5 14:59:01 2022 VERSION=1.1.81 IDTYPE=devname IDNAME=/dev/sdf2 DEVNAME=/dev/sdf2 PVID=yNatmOaFZE0L2UIOv5lmhwHfqtAwYfhq PART=2 IDTYPE=devname IDNAME=/dev/sdd DEVNAME=/dev/sdd PVID=. IDTYPE=devname IDNAME=/dev/sde DEVNAME=/dev/sde PVID=. IDTYPE=devname IDNAME=/dev/sdb DEVNAME=/dev/sdb PVID=. IDTYPE=devname IDNAME=/dev/sdc DEVNAME=/dev/sdc PVID=. IDTYPE=devname IDNAME=. DEVNAME=/dev/sda PVID=Sio0VbF2Of6IVA4DVIe8ycUFXv1MzHz1 [root@localhost ~]# pvcreate /dev/sda Physical volume "/dev/sda" successfully created. [root@localhost ~]# wipefs -a /dev/sda /dev/sda: 8 bytes were erased at offset 0x00000218 (LVM2_member): 4c 56 4d 32 20 30 30 31 [root@localhost ~]# pvs Devices file PVID Sio0VbF2Of6IVA4DVIe8ycUFXv1MzHz1 last seen on /dev/sda not found. Devices file PVID zGc3ox9a3M8FDtBgCMlkkwxgYoSn5TlI not found on device /dev/sda. PV VG Fmt Attr PSize PFree /dev/sdf2 rhel lvm2 a-- <9.00g 0 [root@localhost ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command pvs pid 4966 at Tue Apr 5 14:59:36 2022 VERSION=1.1.83 IDTYPE=devname IDNAME=/dev/sdf2 DEVNAME=/dev/sdf2 PVID=yNatmOaFZE0L2UIOv5lmhwHfqtAwYfhq PART=2 IDTYPE=devname IDNAME=/dev/sdd DEVNAME=/dev/sdd PVID=. IDTYPE=devname IDNAME=/dev/sde DEVNAME=/dev/sde PVID=. IDTYPE=devname IDNAME=/dev/sdb DEVNAME=/dev/sdb PVID=. IDTYPE=devname IDNAME=/dev/sdc DEVNAME=/dev/sdc PVID=. IDTYPE=devname IDNAME=. DEVNAME=/dev/sda PVID=Sio0VbF2Of6IVA4DVIe8ycUFXv1MzHz1 IDTYPE=devname IDNAME=. DEVNAME=/dev/sda PVID=zGc3ox9a3M8FDtBgCMlkkwxgYoSn5TlI Actual results: Error presented after using wipefs on the PV. Expected results: Unsure; this BZ is to evaluate pros/cons of keeping the message
Looks like running `pvscan` fixes the issue with `pvs` command. [root@localhost ~]# pvcreate /dev/sdb Physical volume "/dev/sdb" successfully created. [root@localhost ~]# pvs PV VG Fmt Attr PSize PFree /dev/sda2 rhel9vg lvm2 a-- 90.00g 9.00g /dev/sdb lvm2 --- 80.00g 80.00g [root@localhost ~]# wipefs /dev/sdb DEVICE OFFSET TYPE UUID LABEL sdb 0x218 LVM2_member UOX99o-slaN-cwOO-opL1-gf0Y-WbdU-foKWO5 [root@localhost ~]# wipefs -o 0x218 /dev/sdb /dev/sdb: 8 bytes were erased at offset 0x00000218 (LVM2_member): 4c 56 4d 32 20 30 30 31 [root@localhost ~]# pvs Device /dev/sdb has no PVID (devices file UOX99oslaNcwOOopL1gf0YWbdUfoKWO5) <--- error PV VG Fmt Attr PSize PFree /dev/sda2 rhel9vg lvm2 a-- 90.00g 9.00g [root@localhost ~]# lvs LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert root rhel9vg -wi-ao---- 80.00g swap rhel9vg -wi-ao---- 1.00g [root@localhost ~]# pvscan PV /dev/sda2 VG rhel9vg lvm2 [90.00 GiB / 9.00 GiB free] Total: 1 [90.00 GiB] / in use: 1 [90.00 GiB] / in no VG: 0 [0 ] [root@localhost ~]# pvs PV VG Fmt Attr PSize PFree /dev/sda2 rhel9vg lvm2 a-- 90.00g 9.00g [root@localhost ~]# ------------------------------ [root@localhost ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command pvs pid 3736 at Tue Sep 13 11:40:55 2022 VERSION=1.1.4 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB1f977223-f9262068_ DEVNAME=/dev/sda2 PVID=q2hFg8yCsV0AMyHVKom4KJVto8YuI1vS PART=2 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VBb409e464-4760c654_ DEVNAME=/dev/sdb PVID=. [root@localhost ~]# ------------------------------ [root@localhost ~]# pvremove /dev/sdb No PV found on device /dev/sdb. <--- this is expected but the entry from system.devices is not removed [root@localhost ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command pvs pid 3736 at Tue Sep 13 11:40:55 2022 VERSION=1.1.4 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB1f977223-f9262068_ DEVNAME=/dev/sda2 PVID=q2hFg8yCsV0AMyHVKom4KJVto8YuI1vS PART=2 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VBb409e464-4760c654_ DEVNAME=/dev/sdb PVID=. [root@localhost ~]#
[root@localhost ~]# pvcreate /dev/sdb Physical volume "/dev/sdb" successfully created. [root@localhost ~]# wipefs /dev/sdb DEVICE OFFSET TYPE UUID LABEL sdb 0x218 LVM2_member N6vGB6-BXCL-ykft-9xRb-dS5f-ftEc-CM6zMm [root@localhost ~]# wipefs -a /dev/sdb /dev/sdb: 8 bytes were erased at offset 0x00000218 (LVM2_member): 4c 56 4d 32 20 30 30 31 [root@localhost ~]# pvs Device /dev/sdb has no PVID (devices file N6vGB6BXCLykft9xRbdS5fftEcCM6zMm) PV VG Fmt Attr PSize PFree /dev/sda2 rhel9vg lvm2 a-- 90.00g 9.00g [root@localhost ~]# pvscan PV /dev/sda2 VG rhel9vg lvm2 [90.00 GiB / 9.00 GiB free] Total: 1 [90.00 GiB] / in use: 1 [90.00 GiB] / in no VG: 0 [0 ] [root@localhost ~]# pvs PV VG Fmt Attr PSize PFree /dev/sda2 rhel9vg lvm2 a-- 90.00g 9.00g [root@localhost ~]# [root@localhost ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command pvs pid 4410 at Tue Sep 13 12:03:38 2022 VERSION=1.1.6 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB1f977223-f9262068_ DEVNAME=/dev/sda2 PVID=q2hFg8yCsV0AMyHVKom4KJVto8YuI1vS PART=2 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VBb409e464-4760c654_ DEVNAME=/dev/sdb PVID=. [root@localhost ~]# pvscan --cache pvscan[4421] PV /dev/sda2 online. [root@localhost ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command pvs pid 4410 at Tue Sep 13 12:03:38 2022 VERSION=1.1.6 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB1f977223-f9262068_ DEVNAME=/dev/sda2 PVID=q2hFg8yCsV0AMyHVKom4KJVto8YuI1vS PART=2 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VBb409e464-4760c654_ DEVNAME=/dev/sdb PVID=. [root@localhost ~]# [root@localhost ~]# pvs PV VG Fmt Attr PSize PFree /dev/sda2 rhel9vg lvm2 a-- 90.00g 9.00g
The results you see in comments 4 and 5 benefit from the fact that the device has a wwid. This allows lvm to keep the devices file entry updated without using the PVID. In the original description, lvm didn't have a wwid for the device, so it was using the dev name itself as the id. In this case, lvm doesn't know if the missing PVID was erased with wipefs, or if it's just disconnected from the machine. We've made some effort to improve the error messages around this, but I think there are still some noisy warnings that we want to reduce.
Putting off further changes until we have some specific examples or reports of issues/confusion around this.
I encountered this issue after I upgraded from RHEL 9.1 to RHEL 9.2 and the workaround below worked for me. While awaiting a fix this can be bypassed by setting use_devicesfile=0 inside of /etc/lvm/lvm.conf | https://serverfault.com/questions/1093110/linux-lvm-pvid-last-seen-on | | https://www.burakkurc.com.tr/en/2023/01/29/rhel-9-lvm-devices-file-sys_wwid-nvme-pvid-last-seen-on-not-found |
(In reply to Marcelo Marques from comment #8) > I encountered this issue after I upgraded from RHEL 9.1 to RHEL 9.2 and the > workaround below worked for me. > While awaiting a fix this can be bypassed by setting use_devicesfile=0 > inside of /etc/lvm/lvm.conf > | https://serverfault.com/questions/1093110/linux-lvm-pvid-last-seen-on | > | > https://www.burakkurc.com.tr/en/2023/01/29/rhel-9-lvm-devices-file-sys_wwid- > nvme-pvid-last-seen-on-not-found | I don't think the problem you're describing is related to this bug (although the same messages may be seen.) The info you posted shows that your devices has a real WWID (eui...) so it's surprising that the device wouldn't be found after an upgrade. It would be helpful if you could recreate /etc/lvm/devices/system.devices on 9.2 and show us what it contains (to compare to the non-working info that you posted.) Set use_devicesfile=1 in lvm.conf and run "vgimportdevices -a" to recreate system.devices. Thanks
(In reply to David Teigland from comment #9) > (In reply to Marcelo Marques from comment #8) > > I encountered this issue after I upgraded from RHEL 9.1 to RHEL 9.2 and the > > workaround below worked for me. > > While awaiting a fix this can be bypassed by setting use_devicesfile=0 > > inside of /etc/lvm/lvm.conf > > | https://serverfault.com/questions/1093110/linux-lvm-pvid-last-seen-on | > > | > > https://www.burakkurc.com.tr/en/2023/01/29/rhel-9-lvm-devices-file-sys_wwid- > > nvme-pvid-last-seen-on-not-found | > > I don't think the problem you're describing is related to this bug (although > the same messages may be seen.) The info you posted shows that your devices > has a real WWID (eui...) so it's surprising that the device wouldn't be > found after an upgrade. > > It would be helpful if you could recreate /etc/lvm/devices/system.devices on > 9.2 and show us what it contains (to compare to the non-working info that > you posted.) Set use_devicesfile=1 in lvm.conf and run "vgimportdevices -a" > to recreate system.devices. Thanks Hello David, I tried your suggestion and it worked fine, see details below. Workaround: Set use_devicesfile=1 in /etc/lvm/lvm.conf and run "vgimportdevices -a" to recreate /etc/lvm/devices/system.devices [root@rhel9-pg14 ~]# cat /etc/redhat-release Red Hat Enterprise Linux release 9.2 (Plow) [root@rhel9-pg14 ~]# cat /etc/fstab # # /etc/fstab # Created by anaconda on Fri May 19 22:20:14 2023 # # Accessible filesystems, by reference, are maintained under '/dev/disk/'. # See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info. # # After editing this file, run 'systemctl daemon-reload' to update systemd # units generated from this file. # /dev/mapper/rhel-root / xfs defaults 0 0 UUID=c9f84aac-2dc6-4d00-8826-8e7a883d50d1 /boot xfs defaults 0 0 /dev/mapper/rhel-home /home xfs defaults 0 0 /dev/mapper/rhel-swap none swap defaults 0 0 /dev/vgswap1/lvswap1 swap swap defaults 0 0 /dev/vgpgapp/lvpgapp /pghome/app xfs defaults,noatime 1 1 /dev/vgpgdb/lvpgdb /pghome/db xfs defaults,noatime 1 1 [root@rhel9-pg14 ~]# [root@rhel9-pg14 ~]# vi /etc/lvm/lvm.conf [root@rhel9-pg14 ~]# cat /etc/lvm/lvm.conf | grep use_devicesfile # Configuration option devices/use_devicesfile. # use_devicesfile = 1 use_devicesfile=1 [root@rhel9-pg14 ~]# df -kh Filesystem Size Used Avail Use% Mounted on devtmpfs 4.0M 0 4.0M 0% /dev tmpfs 7.7G 204K 7.7G 1% /dev/shm tmpfs 3.1G 9.2M 3.1G 1% /run /dev/mapper/rhel-root 227G 6.2G 221G 3% / /dev/sda1 7.0G 427M 6.6G 6% /boot /dev/mapper/vgpgapp-lvpgapp 300G 2.2G 298G 1% /pghome/app /dev/mapper/vgpgdb-lvpgdb 300G 2.2G 298G 1% /pghome/db /dev/mapper/rhel-home 50G 392M 50G 1% /home tmpfs 1.6G 52K 1.6G 1% /run/user/42 tmpfs 1.6G 36K 1.6G 1% /run/user/1000 [root@rhel9-pg14 ~]# vgscan Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB35e667ed-46f5a775_ PVID TkbyKnlsMv1d5OCVpf6xO hTHaaGv6waA last seen on /dev/sda2 not found. Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB0c721552-7168bcae_ PVID j4qSIuzDz31MQCeyNxI22 72FWhx37szk last seen on /dev/sdb1 not found. Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB3ced0fad-02e5565d_ PVID LqLunAmQTPb0IEfPZoNqZ 908VxzNgZTm last seen on /dev/sdd1 not found. Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB5c26628f-ec063254_ PVID be06K7vr4SbQSGPd0A01r pMl9y2ODUgb last seen on /dev/sdc1 not found. [root@rhel9-pg14 ~]# [root@rhel9-pg14 ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command vgs pid 1472 at Sat May 20 15:13:16 2023 VERSION=1.1.8 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB35e667ed-46f5a775_ DEVNAME=/dev/sda2 PVID=TkbyKnlsMv1d5OCVpf6xOhTHaaGv6waA PART=2 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB0c721552-7168bcae_ DEVNAME=/dev/sdb1 PVID=j4qSIuzDz31MQCeyNxI2272FWhx37szk PART=1 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB3ced0fad-02e5565d_ DEVNAME=/dev/sdd1 PVID=LqLunAmQTPb0IEfPZoNqZ908VxzNgZTm PART=1 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB5c26628f-ec063254_ DEVNAME=/dev/sdc1 PVID=be06K7vr4SbQSGPd0A01rpMl9y2ODUgb PART=1 [root@rhel9-pg14 ~]# [root@rhel9-pg14 ~]# vgimportdevices -a WARNING: adding device /dev/sdd1 with PVID LqLunAmQTPb0IEfPZoNqZ908VxzNgZTm which is already used for missing device device_id t10.ATA_____VBOX_HARDDISK______ _____________________VB3ced0fad-02e5565d_. Add device with duplicate PV to devices file?yes WARNING: adding device /dev/sdc1 with PVID be06K7vr4SbQSGPd0A01rpMl9y2ODUgb which is already used for missing device device_id t10.ATA_____VBOX_HARDDISK______ _____________________VB5c26628f-ec063254_. Add device with duplicate PV to devices file?yes WARNING: adding device /dev/sdb1 with PVID j4qSIuzDz31MQCeyNxI2272FWhx37szk which is already used for missing device device_id t10.ATA_____VBOX_HARDDISK______ _____________________VB0c721552-7168bcae_. Add device with duplicate PV to devices file?yes WARNING: adding device /dev/sda2 with PVID TkbyKnlsMv1d5OCVpf6xOhTHaaGv6waA which is already used for missing device device_id t10.ATA_____VBOX_HARDDISK______ _____________________VB35e667ed-46f5a775_. Add device with duplicate PV to devices file?yes Added 4 devices to devices file. [root@rhel9-pg14 ~]# [root@rhel9-pg14 ~]# vgscan Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB35e667ed-46f5a775_ PVID TkbyKnlsMv1d5OCVpf6xOhTHaaGv6waA last seen on /dev/sda2 not found. Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB0c721552-7168bcae_ PVID j4qSIuzDz31MQCeyNxI2272FWhx37szk last seen on /dev/sdb1 not found. Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB3ced0fad-02e5565d_ PVID LqLunAmQTPb0IEfPZoNqZ908VxzNgZTm last seen on /dev/sdd1 not found. Devices file sys_wwid t10.ATA_____VBOX_HARDDISK___________________________VB5c26628f-ec063254_ PVID be06K7vr4SbQSGPd0A01rpMl9y2ODUgb last seen on /dev/sdc1 not found. Found volume group "vgpgapp" using metadata type lvm2 Found volume group "vgpgdb" using metadata type lvm2 Found volume group "vgswap1" using metadata type lvm2 Found volume group "rhel" using metadata type lvm2 [root@rhel9-pg14 ~]# [root@rhel9-pg14 ~]# vi /etc/lvm/devices/system.devices --deleted the lines below and saved the file IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB35e667ed-46f5a775_ DEVNAME=/dev/sda2 PVID=TkbyKnlsMv1d5OCVpf6xOhTHaaGv6waA PART=2 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB0c721552-7168bcae_ DEVNAME=/dev/sdb1 PVID=j4qSIuzDz31MQCeyNxI2272FWhx37szk PART=1 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB3ced0fad-02e5565d_ DEVNAME=/dev/sdd1 PVID=LqLunAmQTPb0IEfPZoNqZ908VxzNgZTm PART=1 IDTYPE=sys_wwid IDNAME=t10.ATA_____VBOX_HARDDISK___________________________VB5c26628f-ec063254_ DEVNAME=/dev/sdc1 PVID=be06K7vr4SbQSGPd0A01rpMl9y2ODUgb PART=1 :wq! [root@rhel9-pg14 ~]# cat /etc/lvm/devices/system.devices # LVM uses devices listed in this file. # Created by LVM command vgimportdevices pid 43309 at Mon May 22 12:53:13 2023 VERSION=1.1.9 IDTYPE=sys_wwid IDNAME=t10.ATA_VBOX_HARDDISK_VB3ced0fad-02e5565d DEVNAME=/dev/sdd1 PVID=LqLunAmQTPb0IEfPZoNqZ908VxzNgZTm PART=1 IDTYPE=sys_wwid IDNAME=t10.ATA_VBOX_HARDDISK_VB5c26628f-ec063254 DEVNAME=/dev/sdc1 PVID=be06K7vr4SbQSGPd0A01rpMl9y2ODUgb PART=1 IDTYPE=sys_wwid IDNAME=t10.ATA_VBOX_HARDDISK_VB0c721552-7168bcae DEVNAME=/dev/sdb1 PVID=j4qSIuzDz31MQCeyNxI2272FWhx37szk PART=1 IDTYPE=sys_wwid IDNAME=t10.ATA_VBOX_HARDDISK_VB35e667ed-46f5a775 DEVNAME=/dev/sda2 PVID=TkbyKnlsMv1d5OCVpf6xOhTHaaGv6waA PART=2 [root@rhel9-pg14 ~]# vgscan Found volume group "vgpgapp" using metadata type lvm2 Found volume group "vgpgdb" using metadata type lvm2 Found volume group "vgswap1" using metadata type lvm2 Found volume group "rhel" using metadata type lvm2 [root@rhel9-pg14 ~]# reboot --RHEL 9.2 reboots just fine now and the LVMs are mounted after the reboot without any issue. Thanks, | Marcelo Marques | Esri PS Products | Principal Product Engineer | | Cloud & Database Administrator | OCP – Oracle Certified Professional | | Esri | 380 New York St | Redlands, CA 92373 | USA | | T +909-369-1787 | M +909-255-2079 | mmarques | esri.com | | https://www.linkedin.com/in/mmarquesbr/ |
> Devices file sys_wwid > t10.ATA_____VBOX_HARDDISK___________________________VB35e667ed-46f5a775_ Thanks, this issue is bug 2208039. lvm in 9.2 doesn't handle the trailing underscore which represents a trailing space at the end of the wwid.
(In reply to David Teigland from comment #11) > > Devices file sys_wwid > > t10.ATA_____VBOX_HARDDISK___________________________VB35e667ed-46f5a775_ > > Thanks, this issue is bug 2208039. lvm in 9.2 doesn't handle the trailing > underscore which represents a trailing space at the end of the wwid. Thanks, I appreciate you letting me know. -Marcelo :)