Created attachment 512907 [details]
vdsm log

getDeviceList is synchronous. It would have been nice if it was async, but it is not, and I do not think this could change anytime soon.

The  interesting question are: when does it happen? Is it reproducible? And why did the following little vgs command line block forever? Haim, could you reproduce this and have one of our lvm2 friends (agk?) take a look?

/sbin/lvm vgs --config " devices { preferred_names = [\"^/dev/mapper/\"] ignore_suspended_devices=1 write_cache_state=0 filter = [ \"a%/dev/mapper/1Daffi1309719|/dev/mapper/1ISCSI-PROD11309649|/dev/mapper/1ISCSI-PROD31309649|/dev/mapper/36006016066102900103b0e14f3ade011|/dev/mapper/36006016066102900113b0e14f3ade011|/dev/mapper/36006016066102900114986fbf2ade011|/dev/mapper/36006016066102900123b0e14f3ade011|/dev/mapper/36006016066102900124986fbf2ade011|/dev/mapper/36006016066102900133b0e14f3ade011|/dev/mapper/36006016066102900134986fbf2ade011|/dev/mapper/36006016066102900143b0e14f3ade011|/dev/mapper/36006016066102900144986fbf2ade011|/dev/mapper/36006016066102900153b0e14f3ade011|/dev/mapper/36006016066102900154986fbf2ade011|/dev/mapper/36006016066102900163b0e14f3ade011|/dev/mapper/36006016066102900164986fbf2ade011|/dev/mapper/36006016066102900173b0e14f3ade011|/dev/mapper/36006016066102900174986fbf2ade011|/dev/mapper/36006016066102900183b0e14f3ade011|/dev/mapper/36006016066102900184986fbf2ade011|/dev/mapper/36006016066102900184f9fa57ea2e011|/dev/mapper/36006016066102900193b0e14f3ade011|/dev/mapper/36006016066102900194986fbf2ade011|/dev/mapper/36006016066102900194f9fa57ea2e011|/dev/mapper/360060160661029001a3b0e14f3ade011|/dev/mapper/360060160661029001a4986fbf2ade011|/dev/mapper/360060160661029001a4f9fa57ea2e011|/dev/mapper/360060160661029001b3b0e14f3ade011|/dev/mapper/360060160661029001b4986fbf2ade011|/dev/mapper/360060160661029001b4f9fa57ea2e011|/dev/mapper/360060160661029001c3b0e14f3ade011|/dev/mapper/360060160661029001c4986fbf2ade011|/dev/mapper/360060160661029001c4f9fa57ea2e011|/dev/mapper/360060160661029001d3b0e14f3ade011|/dev/mapper/360060160661029001d4986fbf2ade011|/dev/mapper/360060160661029001d4f9fa57ea2e011|/dev/mapper/360060160661029001e3b0e14f3ade011|/dev/mapper/360060160661029001e4f9fa57ea2e011|/dev/mapper/360060160661029001ee6809f7ea2e011|/dev/mapper/360060160661029001f3b0e14f3ade011|/dev/mapper/360060160661029001f4f9fa57ea2e011|/dev/mapper/360060160661029001fe6809f7ea2e011|/dev/mapper/36006016066102900203b0e14f3ade011|/dev/mapper/36006016066102900204f9fa57ea2e011|/dev/mapper/3600601606610290020bca0fc8b95e011|/dev/mapper/3600601606610290020e6809f7ea2e011|/dev/mapper/36006016066102900213b0e14f3ade011|/dev/mapper/3600601606610290021e6809f7ea2e011|/dev/mapper/36006016066102900223b0e14f3ade011|/dev/mapper/3600601606610290022e6809f7ea2e011|/dev/mapper/36006016066102900233b0e14f3ade011|/dev/mapper/3600601606610290023e6809f7ea2e011|/dev/mapper/36006016066102900243b0e14f3ade011|/dev/mapper/3600601606610290024e6809f7ea2e011|/dev/mapper/36006016066102900341493a64a79e011|/dev/mapper/36006016066102900351493a64a79e011|/dev/mapper/36006016066102900361493a64a79e011|/dev/mapper/36006016066102900441dd38b7ea2e011|/dev/mapper/3600601606610290044225381f79fe011|/dev/mapper/36006016066102900451dd38b7ea2e011|/dev/mapper/3600601606610290046bfdc0df3ade011|/dev/mapper/3600601606610290047bfdc0df3ade011|/dev/mapper/3600601606610290048bfdc0df3ade011|/dev/mapper/3600601606610290049bfdc0df3ade011|/dev/mapper/360060160661029004ab16426f3ade011|/dev/mapper/360060160661029004abfdc0df3ade011|/dev/mapper/360060160661029004bb16426f3ade011|/dev/mapper/360060160661029004bbfdc0df3ade011|/dev/mapper/360060160661029004cb16426f3ade011|/dev/mapper/360060160661029004cbfdc0df3ade011|/dev/mapper/360060160661029004db16426f3ade011|/dev/mapper/360060160661029004dbfdc0df3ade011|/dev/mapper/360060160661029004ebfdc0df3ade011|/dev/mapper/360060160661029004fbfdc0df3ade011|/dev/mapper/3600601606610290050bfdc0df3ade011|/dev/mapper/3600601606610290051bfdc0df3ade011|/dev/mapper/3600601606610290052bfdc0df3ade011|/dev/mapper/3600601606610290053bfdc0df3ade011|/dev/mapper/3600601606610290054bfdc0df3ade011|/dev/mapper/3600601606610290055bfdc0df3ade011|/dev/mapper/3600601606610290056bfdc0df3ade011|/dev/mapper/3600601606610290057bfdc0df3ade011|/dev/mapper/3600601606610290058bfdc0df3ade011|/dev/mapper/3600601606610290059bfdc0df3ade011|/dev/mapper/360060160661029005abfdc0df3ade011|/dev/mapper/360060160661029005bbfdc0df3ade011|/dev/mapper/3600601606610290062dda83b849de011|/dev/mapper/3600601606610290063dda83b849de011|/dev/mapper/3600601606610290064dda83b849de011|/dev/mapper/3600601606610290065dda83b849de011|/dev/mapper/3600601606610290066dda83b849de011|/dev/mapper/3600601606610290067dda83b849de011|/dev/mapper/3600601606610290068dda83b849de011|/dev/mapper/3600601606610290069dda83b849de011|/dev/mapper/360060160661029006adda83b849de011|/dev/mapper/360060160661029006bdda83b849de011|/dev/mapper/360060160661029006c965420f3ade011|/dev/mapper/360060160661029006cdda83b849de011|/dev/mapper/360060160661029006d965420f3ade011|/dev/mapper/360060160661029006ddda83b849de011|/dev/mapper/360060160661029006e965420f3ade011|/dev/mapper/360060160661029006f965420f3ade011|/dev/mapper/36006016066102900706966b27ea2e011|/dev/mapper/3600601606610290070965420f3ade011|/dev/mapper/36006016066102900716966b27ea2e011|/dev/mapper/3600601606610290071965420f3ade011|/dev/mapper/36006016066102900726966b27ea2e011|/dev/mapper/36006016066102900736966b27ea2e011|/dev/mapper/36006016066102900746966b27ea2e011|/dev/mapper/36006016066102900756966b27ea2e011|/dev/mapper/36006016066102900760048ac7ea2e011|/dev/mapper/36006016066102900766966b27ea2e011|/dev/mapper/36006016066102900770048ac7ea2e011|/dev/mapper/36006016066102900776966b27ea2e011|/dev/mapper/36006016066102900780048ac7ea2e011|/dev/mapper/36006016066102900786966b27ea2e011|/dev/mapper/36006016066102900790048ac7ea2e011|/dev/mapper/360060160661029007a0048ac7ea2e011|/dev/mapper/360060160661029007b0048ac7ea2e011|/dev/mapper/360060160661029007c0048ac7ea2e011|/dev/mapper/360060160661029007e7163b87ea2e011|/dev/mapper/360060160661029007e98a17ffa81e011|/dev/mapper/360060160661029007edf64997ea2e011|/dev/mapper/360060160661029007f7163b87ea2e011|/dev/mapper/360060160661029007fdf64997ea2e011|/dev/mapper/36006016066102900807163b87ea2e011|/dev/mapper/3600601606610290080df64997ea2e011|/dev/mapper/36006016066102900817163b87ea2e011|/dev/mapper/3600601606610290081df64997ea2e011|/dev/mapper/36006016066102900827163b87ea2e011|/dev/mapper/3600601606610290082df64997ea2e011|/dev/mapper/36006016066102900837163b87ea2e011|/dev/mapper/36006016066102900847163b87ea2e011|/dev/mapper/3600601606610290084f2da07f3ade011|/dev/mapper/36006016066102900857163b87ea2e011|/dev/mapper/3600601606610290085f2da07f3ade011|/dev/mapper/3600601606610290086f2da07f3ade011|/dev/mapper/3600601606610290087f2da07f3ade011|/dev/mapper/3600601606610290088f2da07f3ade011|/dev/mapper/3600601606610290089f2da07f3ade011|/dev/mapper/360060160661029008af2da07f3ade011|/dev/mapper/360060160661029008bf2da07f3ade011|/dev/mapper/360060160661029008cf2da07f3ade011|/dev/mapper/360060160661029008df2da07f3ade011|/dev/mapper/360060160661029008ef2da07f3ade011|/dev/mapper/360060160661029008ff2da07f3ade011|/dev/mapper/3600601606610290090f2da07f3ade011|/dev/mapper/3600601606610290091f2da07f3ade011|/dev/mapper/3600601606610290092f2da07f3ade011|/dev/mapper/3600601606610290093f2da07f3ade011|/dev/mapper/3600601606610290094f2da07f3ade011|/dev/mapper/3600601606610290095f2da07f3ade011|/dev/mapper/3600601606610290096f2da07f3ade011|/dev/mapper/3600601606610290097f2da07f3ade011|/dev/mapper/3600601606610290098b0c301f3ade011|/dev/mapper/3600601606610290098f2da07f3ade011|/dev/mapper/3600601606610290099b0c301f3ade011|/dev/mapper/360060160661029009a697f653b86e011|/dev/mapper/360060160661029009ab0c301f3ade011|/dev/mapper/360060160661029009bb0c301f3ade011|/dev/mapper/360060160661029009cb0c301f3ade011|/dev/mapper/360060160661029009db0c301f3ade011|/dev/mapper/360060160661029009eb0c301f3ade011|/dev/mapper/360060160661029009fb0c301f3ade011|/dev/mapper/36006016066102900a0b0c301f3ade011|/dev/mapper/36006016066102900a1b0c301f3ade011|/dev/mapper/36006016066102900a2b0c301f3ade011|/dev/mapper/36006016066102900a3b0c301f3ade011|/dev/mapper/36006016066102900a4b0c301f3ade011|/dev/mapper/36006016066102900a5b0c301f3ade011|/dev/mapper/36006016066102900a6b0c301f3ade011|/dev/mapper/36006016066102900a7b0c301f3ade011|/dev/mapper/36006016066102900b8f0591af3ade011|/dev/mapper/36006016066102900b9f0591af3ade011|/dev/mapper/36006016066102900baf0591af3ade011|/dev/mapper/36006016066102900bbf0591af3ade011|/dev/mapper/36006016066102900bcf0591af3ade011|/dev/mapper/36006016066102900bdf0591af3ade011|/dev/mapper/36006016066102900be8c283b0344e011|/dev/mapper/36006016066102900bef0591af3ade011|/dev/mapper/36006016066102900bff0591af3ade011|/dev/mapper/36006016066102900d05093aac08ae011|/dev/mapper/36006016066102900d2bb68927ea2e011|/dev/mapper/36006016066102900d3bb68927ea2e011|/dev/mapper/36006016066102900d4bb68927ea2e011%\", \"r%.*%\" ] }  global {  locking_type=1  prioritise_write_locks=1  wait_for_locks=1 }  backup {  retain_min = 50  retain_days = 0 } " --noheadings --units b --nosuffix --separator | -o uuid,name,attr,size,free,extent_size,extent_count,free_count,tags

(In reply to comment #3)
> getDeviceList is synchronous. It would have been nice if it was async, but it
> is not, and I do not think this could change anytime soon.
> 
> The  interesting question are: when does it happen? Is it reproducible? And why
> did the following little vgs command line block forever? Haim, could you
> reproduce this and have one of our lvm2 friends (agk?) take a look?
> 
> /sbin/lvm vgs --config " devices { preferred_names = [\"^/dev/mapper/\"]
> ignore_suspended_devices=1 write_cache_state=0 filter = [
> \"a%/dev/mapper/1Daffi1309719|/dev/mapper/1ISCSI-PROD11309649|/dev/mapper/1ISCSI-PROD31309649|/dev/mapper/36006016066102900103b0e14f3ade011|/dev/mapper/36006016066102900113b0e14f3ade011|/dev/mapper/36006016066102900114986fbf2ade011|/dev/mapper/36006016066102900123b0e14f3ade011|/dev/mapper/36006016066102900124986fbf2ade011|/dev/mapper/36006016066102900133b0e14f3ade011|/dev/mapper/36006016066102900134986fbf2ade011|/dev/mapper/36006016066102900143b0e14f3ade011|/dev/mapper/36006016066102900144986fbf2ade011|/dev/mapper/36006016066102900153b0e14f3ade011|/dev/mapper/36006016066102900154986fbf2ade011|/dev/mapper/36006016066102900163b0e14f3ade011|/dev/mapper/36006016066102900164986fbf2ade011|/dev/mapper/36006016066102900173b0e14f3ade011|/dev/mapper/36006016066102900174986fbf2ade011|/dev/mapper/36006016066102900183b0e14f3ade011|/dev/mapper/36006016066102900184986fbf2ade011|/dev/mapper/36006016066102900184f9fa57ea2e011|/dev/mapper/36006016066102900193b0e14f3ade011|/dev/mapper/36006016066102900194986fbf2ade011|/dev/mapper/36006016066102900194f9fa57ea2e011|/dev/mapper/360060160661029001a3b0e14f3ade011|/dev/mapper/360060160661029001a4986fbf2ade011|/dev/mapper/360060160661029001a4f9fa57ea2e011|/dev/mapper/360060160661029001b3b0e14f3ade011|/dev/mapper/360060160661029001b4986fbf2ade011|/dev/mapper/360060160661029001b4f9fa57ea2e011|/dev/mapper/360060160661029001c3b0e14f3ade011|/dev/mapper/360060160661029001c4986fbf2ade011|/dev/mapper/360060160661029001c4f9fa57ea2e011|/dev/mapper/360060160661029001d3b0e14f3ade011|/dev/mapper/360060160661029001d4986fbf2ade011|/dev/mapper/360060160661029001d4f9fa57ea2e011|/dev/mapper/360060160661029001e3b0e14f3ade011|/dev/mapper/360060160661029001e4f9fa57ea2e011|/dev/mapper/360060160661029001ee6809f7ea2e011|/dev/mapper/360060160661029001f3b0e14f3ade011|/dev/mapper/360060160661029001f4f9fa57ea2e011|/dev/mapper/360060160661029001fe6809f7ea2e011|/dev/mapper/36006016066102900203b0e14f3ade011|/dev/mapper/36006016066102900204f9fa57ea2e011|/dev/mapper/3600601606610290020bca0fc8b95e011|/dev/mapper/3600601606610290020e6809f7ea2e011|/dev/mapper/36006016066102900213b0e14f3ade011|/dev/mapper/3600601606610290021e6809f7ea2e011|/dev/mapper/36006016066102900223b0e14f3ade011|/dev/mapper/3600601606610290022e6809f7ea2e011|/dev/mapper/36006016066102900233b0e14f3ade011|/dev/mapper/3600601606610290023e6809f7ea2e011|/dev/mapper/36006016066102900243b0e14f3ade011|/dev/mapper/3600601606610290024e6809f7ea2e011|/dev/mapper/36006016066102900341493a64a79e011|/dev/mapper/36006016066102900351493a64a79e011|/dev/mapper/36006016066102900361493a64a79e011|/dev/mapper/36006016066102900441dd38b7ea2e011|/dev/mapper/3600601606610290044225381f79fe011|/dev/mapper/36006016066102900451dd38b7ea2e011|/dev/mapper/3600601606610290046bfdc0df3ade011|/dev/mapper/3600601606610290047bfdc0df3ade011|/dev/mapper/3600601606610290048bfdc0df3ade011|/dev/mapper/3600601606610290049bfdc0df3ade011|/dev/mapper/360060160661029004ab16426f3ade011|/dev/mapper/360060160661029004abfdc0df3ade011|/dev/mapper/360060160661029004bb16426f3ade011|/dev/mapper/360060160661029004bbfdc0df3ade011|/dev/mapper/360060160661029004cb16426f3ade011|/dev/mapper/360060160661029004cbfdc0df3ade011|/dev/mapper/360060160661029004db16426f3ade011|/dev/mapper/360060160661029004dbfdc0df3ade011|/dev/mapper/360060160661029004ebfdc0df3ade011|/dev/mapper/360060160661029004fbfdc0df3ade011|/dev/mapper/3600601606610290050bfdc0df3ade011|/dev/mapper/3600601606610290051bfdc0df3ade011|/dev/mapper/3600601606610290052bfdc0df3ade011|/dev/mapper/3600601606610290053bfdc0df3ade011|/dev/mapper/3600601606610290054bfdc0df3ade011|/dev/mapper/3600601606610290055bfdc0df3ade011|/dev/mapper/3600601606610290056bfdc0df3ade011|/dev/mapper/3600601606610290057bfdc0df3ade011|/dev/mapper/3600601606610290058bfdc0df3ade011|/dev/mapper/3600601606610290059bfdc0df3ade011|/dev/mapper/360060160661029005abfdc0df3ade011|/dev/mapper/360060160661029005bbfdc0df3ade011|/dev/mapper/3600601606610290062dda83b849de011|/dev/mapper/3600601606610290063dda83b849de011|/dev/mapper/3600601606610290064dda83b849de011|/dev/mapper/3600601606610290065dda83b849de011|/dev/mapper/3600601606610290066dda83b849de011|/dev/mapper/3600601606610290067dda83b849de011|/dev/mapper/3600601606610290068dda83b849de011|/dev/mapper/3600601606610290069dda83b849de011|/dev/mapper/360060160661029006adda83b849de011|/dev/mapper/360060160661029006bdda83b849de011|/dev/mapper/360060160661029006c965420f3ade011|/dev/mapper/360060160661029006cdda83b849de011|/dev/mapper/360060160661029006d965420f3ade011|/dev/mapper/360060160661029006ddda83b849de011|/dev/mapper/360060160661029006e965420f3ade011|/dev/mapper/360060160661029006f965420f3ade011|/dev/mapper/36006016066102900706966b27ea2e011|/dev/mapper/3600601606610290070965420f3ade011|/dev/mapper/36006016066102900716966b27ea2e011|/dev/mapper/3600601606610290071965420f3ade011|/dev/mapper/36006016066102900726966b27ea2e011|/dev/mapper/36006016066102900736966b27ea2e011|/dev/mapper/36006016066102900746966b27ea2e011|/dev/mapper/36006016066102900756966b27ea2e011|/dev/mapper/36006016066102900760048ac7ea2e011|/dev/mapper/36006016066102900766966b27ea2e011|/dev/mapper/36006016066102900770048ac7ea2e011|/dev/mapper/36006016066102900776966b27ea2e011|/dev/mapper/36006016066102900780048ac7ea2e011|/dev/mapper/36006016066102900786966b27ea2e011|/dev/mapper/36006016066102900790048ac7ea2e011|/dev/mapper/360060160661029007a0048ac7ea2e011|/dev/mapper/360060160661029007b0048ac7ea2e011|/dev/mapper/360060160661029007c0048ac7ea2e011|/dev/mapper/360060160661029007e7163b87ea2e011|/dev/mapper/360060160661029007e98a17ffa81e011|/dev/mapper/360060160661029007edf64997ea2e011|/dev/mapper/360060160661029007f7163b87ea2e011|/dev/mapper/360060160661029007fdf64997ea2e011|/dev/mapper/36006016066102900807163b87ea2e011|/dev/mapper/3600601606610290080df64997ea2e011|/dev/mapper/36006016066102900817163b87ea2e011|/dev/mapper/3600601606610290081df64997ea2e011|/dev/mapper/36006016066102900827163b87ea2e011|/dev/mapper/3600601606610290082df64997ea2e011|/dev/mapper/36006016066102900837163b87ea2e011|/dev/mapper/36006016066102900847163b87ea2e011|/dev/mapper/3600601606610290084f2da07f3ade011|/dev/mapper/36006016066102900857163b87ea2e011|/dev/mapper/3600601606610290085f2da07f3ade011|/dev/mapper/3600601606610290086f2da07f3ade011|/dev/mapper/3600601606610290087f2da07f3ade011|/dev/mapper/3600601606610290088f2da07f3ade011|/dev/mapper/3600601606610290089f2da07f3ade011|/dev/mapper/360060160661029008af2da07f3ade011|/dev/mapper/360060160661029008bf2da07f3ade011|/dev/mapper/360060160661029008cf2da07f3ade011|/dev/mapper/360060160661029008df2da07f3ade011|/dev/mapper/360060160661029008ef2da07f3ade011|/dev/mapper/360060160661029008ff2da07f3ade011|/dev/mapper/3600601606610290090f2da07f3ade011|/dev/mapper/3600601606610290091f2da07f3ade011|/dev/mapper/3600601606610290092f2da07f3ade011|/dev/mapper/3600601606610290093f2da07f3ade011|/dev/mapper/3600601606610290094f2da07f3ade011|/dev/mapper/3600601606610290095f2da07f3ade011|/dev/mapper/3600601606610290096f2da07f3ade011|/dev/mapper/3600601606610290097f2da07f3ade011|/dev/mapper/3600601606610290098b0c301f3ade011|/dev/mapper/3600601606610290098f2da07f3ade011|/dev/mapper/3600601606610290099b0c301f3ade011|/dev/mapper/360060160661029009a697f653b86e011|/dev/mapper/360060160661029009ab0c301f3ade011|/dev/mapper/360060160661029009bb0c301f3ade011|/dev/mapper/360060160661029009cb0c301f3ade011|/dev/mapper/360060160661029009db0c301f3ade011|/dev/mapper/360060160661029009eb0c301f3ade011|/dev/mapper/360060160661029009fb0c301f3ade011|/dev/mapper/36006016066102900a0b0c301f3ade011|/dev/mapper/36006016066102900a1b0c301f3ade011|/dev/mapper/36006016066102900a2b0c301f3ade011|/dev/mapper/36006016066102900a3b0c301f3ade011|/dev/mapper/36006016066102900a4b0c301f3ade011|/dev/mapper/36006016066102900a5b0c301f3ade011|/dev/mapper/36006016066102900a6b0c301f3ade011|/dev/mapper/36006016066102900a7b0c301f3ade011|/dev/mapper/36006016066102900b8f0591af3ade011|/dev/mapper/36006016066102900b9f0591af3ade011|/dev/mapper/36006016066102900baf0591af3ade011|/dev/mapper/36006016066102900bbf0591af3ade011|/dev/mapper/36006016066102900bcf0591af3ade011|/dev/mapper/36006016066102900bdf0591af3ade011|/dev/mapper/36006016066102900be8c283b0344e011|/dev/mapper/36006016066102900bef0591af3ade011|/dev/mapper/36006016066102900bff0591af3ade011|/dev/mapper/36006016066102900d05093aac08ae011|/dev/mapper/36006016066102900d2bb68927ea2e011|/dev/mapper/36006016066102900d3bb68927ea2e011|/dev/mapper/36006016066102900d4bb68927ea2e011%\",
> \"r%.*%\" ] }  global {  locking_type=1  prioritise_write_locks=1 
> wait_for_locks=1 }  backup {  retain_min = 50  retain_days = 0 } " --noheadings
> --units b --nosuffix --separator | -o
> uuid,name,attr,size,free,extent_size,extent_count,free_count,tags

Dan, first, I already spoke with LVM guys (mbroz), and the reason lvm operation stuck was due to multipath errors along with kpartx stuck in kernel. 
I think i know how to reproduce, and might open a different bz on device-mapper-multipith, however, it does not change the fact that getDeviceList is sync operation, and if it stuck, i want to know about it, and maybe build some login around it from backend side, of course, this can be postpone to later versions.

I have no idea how rhev-m could circumvent a call to getDeviceList that takes a lot of time; but making this verb asynchronous (in some future version) sounds reasonable.

Closing old bugs. If this issue is still relevant/important in current version, please re-open the bug.