Bug 2240011

(undercloud) [lab] stack@wallaby-uc:~$ ansible -i ~/overcloud-deploy/wallaby/tripleo-ansible-inventory.yaml Controller,ComputeOvsDpdkSriov -m shell -b -a 'grep -r req-60fb13c3-24b6-45c4-82bc-0298d27e17cf /var/log/containers|grep -i numa'
wallaby-ctrl2 | FAILED | rc=1 >>
non-zero return code
wallaby-ctrl1 | CHANGED | rc=0 >>
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:06.680 14 DEBUG nova.scheduler.host_manager [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Update host state from compute node: ComputeNode(cpu_allocation_ratio=16.0,cpu_info='{"arch": "x86_64", "model": "Broadwell-noTSX-IBRS", "vendor": "Intel", "topology": {"cells": 1, "sockets": 8, "cores": 1, "threads": 1}, "features": ["movbe", "hypervisor", "sse", "rdctl-no", "pdpe1gb", "msr", "xsavec", "pse", "sep", "stibp", "sse4.1", "nx", "mca", "sse2", "amd-ssbd", "ibpb", "mds-no", "abm", "bmi2", "smep", "lahf_lm", "vmx", "x2apic", "rdtscp", "md-clear", "adx", "ibrs-all", "rdrand", "f16c", "mce", "vme", "ssse3", "fsgsbase", "clflush", "apic", "fxsr", "pge", "mmx", "xgetbv1", "popcnt", "pni", "clflushopt", "sse4.2", "pclmuldq", "xsaves", "pae", "3dnowprefetch", "ibrs", "mtrr", "pat", "ssbd", "tsc-deadline", "cx16", "smap", "fma", "cx8", "pschange-mc-no", "rdseed", "cmov", "erms", "syscall", "lm", "amd-stibp", "de", "aes", "avx2", "bmi1", "tsx-ctrl", "tsc_adjust", "ss", "arat", "pcid", "avx", "fpu", "pse36", "skip-l1dfl-vmentry", "xsaveopt", "spec-ctrl", "umip", "tsc", "arch-capabilities", "xsave", "invpcid"]}',created_at=2023-09-20T10:46:52Z,current_workload=0,deleted=False,deleted_at=None,disk_allocation_ratio=1.0,disk_available_least=554,free_disk_gb=1799,free_ram_mb=27726,host='wallaby-comp0.example.com',host_ip=172.20.55.110,hypervisor_hostname='wallaby-comp0.example.com',hypervisor_type='QEMU',hypervisor_version=7002000,id=2,local_gb=1799,local_gb_used=0,mapped=0,memory_mb=31822,memory_mb_used=4096,metrics='[]',numa_topology='{"nova_object.name": "NUMATopology", "nova_object.namespace": "nova", "nova_object.version": "1.2", "nova_object.data": {"cells": [{"nova_object.name": "NUMACell", "nova_object.namespace": "nova", "nova_object.version": "1.5", "nova_object.data": {"id": 0, "cpuset": [0, 1, 2, 3, 4, 5, 6, 7], "pcpuset": [0, 1, 2, 3, 4, 5, 6, 7], "memory": 31822, "cpu_usage": 0, "memory_usage": 0, "pinned_cpus": [], "siblings": [[6], [2], [5], [4], [1], [7], [0], [3]], "mempages": [{"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 4, "total": 8146582, "used": 0, "reserved": 0}, "nova_object.changes": ["used", "reserved", "total", "size_kb"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 2048, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["used", "reserved", "total", "size_kb"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 1048576, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["used", "reserved", "total", "size_kb"]}], "network_metadata": {"nova_object.name": "NetworkMetadata", "nova_object.namespace": "nova", "nova_object.version": "1.0", "nova_object.data": {"physnets": [], "tunneled": false}, "nova_object.changes": ["tunneled", "physnets"]}, "socket": null}, "nova_object.changes": ["pcpuset", "pinned_cpus", "network_metadata", "siblings", "id", "cpu_usage", "memory", "socket", "memory_usage", "cpuset", "mempages"]}]}, "nova_object.changes": ["cells"]}',pci_device_pools=PciDevicePoolList,ram_allocation_ratio=1.0,running_vms=0,service_id=None,stats={failed_builds='0'},supported_hv_specs=[HVSpec,HVSpec,HVSpec,HVSpec],updated_at=2023-09-21T06:46:12Z,uuid=a6bd0a63-c0e2-4afa-88c1-19a7d62df27f,vcpus=8,vcpus_used=0) _locked_update /usr/lib/python3.9/site-packages/nova/scheduler/host_manager.py:171
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:06.682 14 DEBUG nova.scheduler.host_manager [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Update host state from compute node: ComputeNode(cpu_allocation_ratio=16.0,cpu_info='{"arch": "x86_64", "model": "Broadwell-noTSX-IBRS", "vendor": "Intel", "topology": {"cells": 1, "sockets": 8, "cores": 1, "threads": 1}, "features": ["pse", "pse36", "invpcid", "nx", "arch-capabilities", "arat", "lahf_lm", "popcnt", "fpu", "avx2", "msr", "pge", "pclmuldq", "smap", "abm", "pae", "mca", "cx8", "xsave", "apic", "mmx", "tsc", "hypervisor", "ssse3", "mce", "cmov", "mtrr", "amd-stibp", "md-clear", "ibpb", "f16c", "sse4.2", "syscall", "tsx-ctrl", "ibrs-all", "sse2", "adx", "movbe", "stibp", "bmi2", "amd-ssbd", "skip-l1dfl-vmentry", "tsc_adjust", "rdrand", "xsavec", "bmi1", "rdctl-no", "3dnowprefetch", "clflush", "vme", "rdseed", "ss", "sep", "pdpe1gb", "sse4.1", "xgetbv1", "clflushopt", "fxsr", "smep", "tsc-deadline", "x2apic", "ibrs", "pat", "pschange-mc-no", "mds-no", "fsgsbase", "xsaveopt", "vmx", "umip", "pni", "erms", "sse", "ssbd", "aes", "lm", "fma", "rdtscp", "spec-ctrl", "pcid", "cx16", "avx", "xsaves", "de"]}',created_at=2023-09-20T10:46:53Z,current_workload=0,deleted=False,deleted_at=None,disk_allocation_ratio=1.0,disk_available_least=554,free_disk_gb=1799,free_ram_mb=27726,host='wallaby-comp1.example.com',host_ip=172.20.55.111,hypervisor_hostname='wallaby-comp1.example.com',hypervisor_type='QEMU',hypervisor_version=7002000,id=5,local_gb=1799,local_gb_used=0,mapped=0,memory_mb=31822,memory_mb_used=4096,metrics='[]',numa_topology='{"nova_object.name": "NUMATopology", "nova_object.namespace": "nova", "nova_object.version": "1.2", "nova_object.data": {"cells": [{"nova_object.name": "NUMACell", "nova_object.namespace": "nova", "nova_object.version": "1.5", "nova_object.data": {"id": 0, "cpuset": [0, 1, 2, 3, 4, 5, 6, 7], "pcpuset": [0, 1, 2, 3, 4, 5, 6, 7], "memory": 31822, "cpu_usage": 0, "memory_usage": 0, "pinned_cpus": [], "siblings": [[6], [2], [5], [4], [1], [7], [0], [3]], "mempages": [{"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 4, "total": 8146582, "used": 0, "reserved": 0}, "nova_object.changes": ["size_kb", "reserved", "used", "total"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 2048, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["size_kb", "reserved", "used", "total"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 1048576, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["size_kb", "reserved", "used", "total"]}], "network_metadata": {"nova_object.name": "NetworkMetadata", "nova_object.namespace": "nova", "nova_object.version": "1.0", "nova_object.data": {"physnets": [], "tunneled": false}, "nova_object.changes": ["tunneled", "physnets"]}, "socket": null}, "nova_object.changes": ["socket", "cpuset", "memory", "mempages", "network_metadata", "memory_usage", "siblings", "cpu_usage", "id", "pinned_cpus", "pcpuset"]}]}, "nova_object.changes": ["cells"]}',pci_device_pools=PciDevicePoolList,ram_allocation_ratio=1.0,running_vms=0,service_id=None,stats={failed_builds='0'},supported_hv_specs=[HVSpec,HVSpec,HVSpec,HVSpec],updated_at=2023-09-21T06:46:20Z,uuid=77d8f7b4-8f70-476c-8d2c-deabba95f0ab,vcpus=8,vcpus_used=0) _locked_update /usr/lib/python3.9/site-packages/nova/scheduler/host_manager.py:171
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:06.683 14 DEBUG nova.scheduler.host_manager [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Update host state from compute node: ComputeNode(cpu_allocation_ratio=16.0,cpu_info='{"arch": "x86_64", "model": "Broadwell-noTSX-IBRS", "vendor": "Intel", "topology": {"cells": 1, "sockets": 8, "cores": 1, "threads": 1}, "features": ["pge", "mds-no", "pat", "xsaveopt", "amd-ssbd", "tsc-deadline", "f16c", "pschange-mc-no", "sse", "hypervisor", "cx16", "cmov", "xgetbv1", "tsx-ctrl", "mce", "msr", "vme", "fpu", "ssse3", "lm", "ssbd", "sep", "rdctl-no", "movbe", "de", "pclmuldq", "smap", "umip", "tsc_adjust", "pni", "vmx", "clflush", "smep", "x2apic", "xsaves", "cx8", "arat", "rdseed", "lahf_lm", "popcnt", "pcid", "avx", "sse4.2", "sse4.1", "tsc", "apic", "sse2", "syscall", "rdtscp", "mmx", "pse36", "fsgsbase", "avx2", "nx", "aes", "ibrs-all", "spec-ctrl", "pse", "fxsr", "mtrr", "ibpb", "3dnowprefetch", "amd-stibp", "pae", "arch-capabilities", "xsavec", "skip-l1dfl-vmentry", "bmi2", "invpcid", "fma", "rdrand", "ibrs", "erms", "md-clear", "abm", "clflushopt", "mca", "bmi1", "pdpe1gb", "ss", "adx", "stibp", "xsave"]}',created_at=2023-09-20T10:46:53Z,current_workload=0,deleted=False,deleted_at=None,disk_allocation_ratio=1.0,disk_available_least=554,free_disk_gb=1799,free_ram_mb=25678,host='wallaby-comp2.example.com',host_ip=172.20.55.112,hypervisor_hostname='wallaby-comp2.example.com',hypervisor_type='QEMU',hypervisor_version=7002000,id=8,local_gb=1799,local_gb_used=0,mapped=0,memory_mb=31822,memory_mb_used=6144,metrics='[]',numa_topology='{"nova_object.name": "NUMATopology", "nova_object.namespace": "nova", "nova_object.version": "1.2", "nova_object.data": {"cells": [{"nova_object.name": "NUMACell", "nova_object.namespace": "nova", "nova_object.version": "1.5", "nova_object.data": {"id": 0, "cpuset": [0, 1, 2, 3, 4, 5, 6, 7], "pcpuset": [0, 1, 2, 3, 4, 5, 6, 7], "memory": 31822, "cpu_usage": 0, "memory_usage": 0, "pinned_cpus": [], "siblings": [[6], [2], [5], [4], [1], [7], [0], [3]], "mempages": [{"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 4, "total": 8146582, "used": 0, "reserved": 0}, "nova_object.changes": ["total", "size_kb", "used", "reserved"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 2048, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["total", "size_kb", "used", "reserved"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 1048576, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["total", "size_kb", "used", "reserved"]}], "network_metadata": {"nova_object.name": "NetworkMetadata", "nova_object.namespace": "nova", "nova_object.version": "1.0", "nova_object.data": {"physnets": [], "tunneled": false}, "nova_object.changes": ["tunneled", "physnets"]}, "socket": null}, "nova_object.changes": ["memory", "pinned_cpus", "socket", "cpu_usage", "siblings", "mempages", "network_metadata", "cpuset", "id", "pcpuset", "memory_usage"]}]}, "nova_object.changes": ["cells"]}',pci_device_pools=PciDevicePoolList,ram_allocation_ratio=1.0,running_vms=1,service_id=None,stats={failed_builds='0',io_workload='0',num_instances='1',num_os_type_None='1',num_proj_0b213a7ab3374a90a2d96f38198b4ce2='1',num_task_None='1',num_vm_active='1'},supported_hv_specs=[HVSpec,HVSpec,HVSpec,HVSpec],updated_at=2023-09-21T06:46:17Z,uuid=5c68a739-2908-48a9-870a-4d72193d8605,vcpus=8,vcpus_used=2) _locked_update /usr/lib/python3.9/site-packages/nova/scheduler/host_manager.py:171
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:06.684 14 DEBUG nova.scheduler.host_manager [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Update host state from compute node: ComputeNode(cpu_allocation_ratio=16.0,cpu_info='{"arch": "x86_64", "model": "Haswell-noTSX-IBRS", "vendor": "Intel", "topology": {"cells": 2, "sockets": 1, "cores": 12, "threads": 2}, "features": ["arat", "lm", "fpu", "mca", "nx", "xsave", "ssse3", "smx", "tsc_adjust", "pse", "mmx", "movbe", "de", "rdtscp", "tsc-deadline", "est", "avx", "tm", "fma", "ssbd", "xtpr", "ht", "sse", "pcid", "erms", "ss", "lahf_lm", "rdrand", "tsc", "cx8", "syscall", "pge", "ds_cpl", "sse2", "bmi1", "mce", "f16c", "cmov", "popcnt", "monitor", "cx16", "dtes64", "pae", "tm2", "mtrr", "aes", "spec-ctrl", "pni", "invtsc", "sse4.2", "abm", "fxsr", "sep", "md-clear", "smep", "stibp", "x2apic", "vme", "pse36", "acpi", "bmi2", "dca", "ds", "msr", "vmx", "pdpe1gb", "xsaveopt", "pclmuldq", "apic", "pdcm", "pbe", "pat", "invpcid", "sse4.1", "clflush", "avx2", "fsgsbase"]}',created_at=2023-09-20T10:47:18Z,current_workload=0,deleted=False,deleted_at=None,disk_allocation_ratio=1.0,disk_available_least=554,free_disk_gb=1799,free_ram_mb=119975,host='wallaby-cnfv0.example.com',host_ip=172.20.55.113,hypervisor_hostname='wallaby-cnfv0.example.com',hypervisor_type='QEMU',hypervisor_version=7002000,id=11,local_gb=1799,local_gb_used=0,mapped=0,memory_mb=128167,memory_mb_used=8192,metrics='[]',numa_topology='{"nova_object.name": "NUMATopology", "nova_object.namespace": "nova", "nova_object.version": "1.2", "nova_object.data": {"cells": [{"nova_object.name": "NUMACell", "nova_object.namespace": "nova", "nova_object.version": "1.5", "nova_object.data": {"id": 0, "cpuset": [0, 24], "pcpuset": [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44], "memory": 63703, "cpu_usage": 0, "memory_usage": 0, "pinned_cpus": [], "siblings": [[10, 34], [12, 36], [18, 42], [2, 26], [6, 30], [4, 28], [8, 32], [16, 40], [20, 44], [0, 24], [38, 14]], "mempages": [{"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 4, "total": 1890145, "used": 0, "reserved": 0}, "nova_object.changes": ["size_kb", "used", "total", "reserved"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 2048, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["size_kb", "used", "total", "reserved"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 1048576, "total": 55, "used": 0, "reserved": 8}, "nova_object.changes": ["size_kb", "used", "total", "reserved"]}], "network_metadata": {"nova_object.name": "NetworkMetadata", "nova_object.namespace": "nova", "nova_object.version": "1.0", "nova_object.data": {"physnets": ["sriov0", "sriov1", "datacentre"], "tunneled": true}, "nova_object.changes": ["tunneled", "physnets"]}, "socket": 0}, "nova_object.changes": ["socket", "memory", "cpuset", "memory_usage", "mempages", "network_metadata", "pinned_cpus", "pcpuset", "cpu_usage", "id", "siblings"]}, {"nova_object.name": "NUMACell", "nova_object.namespace": "nova", "nova_object.version": "1.5", "nova_object.data": {"id": 1, "cpuset": [1, 25], "pcpuset": [7, 9, 11, 13, 15, 17, 19, 21, 23, 31, 33, 35, 37, 39, 41, 43, 45, 47], "memory": 64464, "cpu_usage": 0, "memory_usage": 0, "pinned_cpus": [], "siblings": [[13, 37], [23, 47], [7, 31], [9, 33], [17, 41], [19, 43], [1, 25], [11, 35], [15, 39], [21, 45]], "mempages": [{"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 4, "total": 2085004, "used": 0, "reserved": 0}, "nova_object.changes": ["size_kb", "used", "total", "reserved"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 2048, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["size_kb", "used", "total", "reserved"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 1048576, "total": 55, "used": 0, "reserved": 8}, "nova_object.changes": ["size_kb", "used", "total", "reserved"]}], "network_metadata": {"nova_object.name": "NetworkMetadata", "nova_object.namespace": "nova", "nova_object.version": "1.0", "nova_object.data": {"physnets": ["dpdk", "datacentre"], "tunneled": true}, "nova_object.changes": ["tunneled", "physnets"]}, "socket": 1}, "nova_object.changes": ["socket", "memory", "cpuset", "memory_usage", "mempages", "network_metadata", "pinned_cpus", "pcpuset", "cpu_usage", "id", "siblings"]}]}, "nova_object.changes": ["cells"]}',pci_device_pools=PciDevicePoolList,ram_allocation_ratio=1.0,running_vms=0,service_id=None,stats={failed_builds='0'},supported_hv_specs=[HVSpec,HVSpec,HVSpec,HVSpec],updated_at=2023-09-21T06:46:28Z,uuid=c26bcac4-840d-4f82-8b64-f4285bc84210,vcpus=4,vcpus_used=0) _locked_update /usr/lib/python3.9/site-packages/nova/scheduler/host_manager.py:171
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:06.685 14 DEBUG nova.scheduler.host_manager [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Update host state with aggregates: [Aggregate(created_at=2023-09-20T12:09:48Z,deleted=False,deleted_at=None,hosts=['wallaby-cnfv0.example.com','wallaby-cnfv1.example.com'],id=2,metadata={availability_zone='nfv',compute_dpdk='true',trait:COMPUTE_SOCKET_PCI_NUMA_AFFINITY='required'},name='dpdk',updated_at=2023-09-21T06:46:34Z,uuid=f365adfc-92f9-4367-8c6f-a3df69ebec57), Aggregate(created_at=2023-09-20T14:29:18Z,deleted=False,deleted_at=None,hosts=['wallaby-cnfv0.example.com','wallaby-cnfv1.example.com'],id=5,metadata={compute_sriov='true'},name='sriov',updated_at=None,uuid=1a340915-869d-4eaf-a7f8-c5f7ed46dbeb)] _locked_update /usr/lib/python3.9/site-packages/nova/scheduler/host_manager.py:174
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:06.686 14 DEBUG nova.scheduler.host_manager [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Update host state from compute node: ComputeNode(cpu_allocation_ratio=16.0,cpu_info='{"arch": "x86_64", "model": "Haswell-noTSX-IBRS", "vendor": "Intel", "topology": {"cells": 2, "sockets": 1, "cores": 12, "threads": 2}, "features": ["spec-ctrl", "tm", "ss", "tsc_adjust", "popcnt", "cmov", "sep", "ds_cpl", "fsgsbase", "xtpr", "sse4.2", "arat", "fpu", "avx", "mca", "est", "pbe", "de", "smep", "pge", "invpcid", "mce", "ds", "stibp", "abm", "invtsc", "sse", "msr", "xsave", "md-clear", "pse36", "pat", "fxsr", "dtes64", "rdtscp", "movbe", "dca", "aes", "apic", "pae", "acpi", "pdcm", "mmx", "rdrand", "mtrr", "vme", "ht", "nx", "bmi2", "sse4.1", "lm", "cx8", "sse2", "bmi1", "ssse3", "lahf_lm", "pclmuldq", "pni", "tm2", "tsc", "erms", "syscall", "clflush", "avx2", "fma", "x2apic", "pcid", "monitor", "pse", "f16c", "xsaveopt", "ssbd", "smx", "tsc-deadline", "cx16", "vmx", "pdpe1gb"]}',created_at=2023-09-20T10:47:31Z,current_workload=0,deleted=False,deleted_at=None,disk_allocation_ratio=1.0,disk_available_least=554,free_disk_gb=1799,free_ram_mb=119975,host='wallaby-cnfv1.example.com',host_ip=172.20.55.114,hypervisor_hostname='wallaby-cnfv1.example.com',hypervisor_type='QEMU',hypervisor_version=7002000,id=14,local_gb=1799,local_gb_used=0,mapped=0,memory_mb=128167,memory_mb_used=8192,metrics='[]',numa_topology='{"nova_object.name": "NUMATopology", "nova_object.namespace": "nova", "nova_object.version": "1.2", "nova_object.data": {"cells": [{"nova_object.name": "NUMACell", "nova_object.namespace": "nova", "nova_object.version": "1.5", "nova_object.data": {"id": 0, "cpuset": [0, 24], "pcpuset": [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44], "memory": 63703, "cpu_usage": 0, "memory_usage": 0, "pinned_cpus": [], "siblings": [[10, 34], [12, 36], [18, 42], [2, 26], [6, 30], [4, 28], [8, 32], [16, 40], [20, 44], [0, 24], [38, 14]], "mempages": [{"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 4, "total": 1890139, "used": 0, "reserved": 0}, "nova_object.changes": ["reserved", "size_kb", "total", "used"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 2048, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["reserved", "size_kb", "total", "used"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 1048576, "total": 55, "used": 0, "reserved": 8}, "nova_object.changes": ["reserved", "size_kb", "total", "used"]}], "network_metadata": {"nova_object.name": "NetworkMetadata", "nova_object.namespace": "nova", "nova_object.version": "1.0", "nova_object.data": {"physnets": ["sriov0", "datacentre", "sriov1"], "tunneled": true}, "nova_object.changes": ["physnets", "tunneled"]}, "socket": 0}, "nova_object.changes": ["network_metadata", "siblings", "mempages", "memory", "memory_usage", "cpuset", "pinned_cpus", "socket", "id", "pcpuset", "cpu_usage"]}, {"nova_object.name": "NUMACell", "nova_object.namespace": "nova", "nova_object.version": "1.5", "nova_object.data": {"id": 1, "cpuset": [1, 25], "pcpuset": [7, 9, 11, 13, 15, 17, 19, 21, 23, 31, 33, 35, 37, 39, 41, 43, 45, 47], "memory": 64464, "cpu_usage": 0, "memory_usage": 0, "pinned_cpus": [], "siblings": [[13, 37], [23, 47], [7, 31], [9, 33], [17, 41], [19, 43], [1, 25], [11, 35], [15, 39], [21, 45]], "mempages": [{"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 4, "total": 2085004, "used": 0, "reserved": 0}, "nova_object.changes": ["reserved", "size_kb", "total", "used"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 2048, "total": 0, "used": 0, "reserved": 0}, "nova_object.changes": ["reserved", "size_kb", "total", "used"]}, {"nova_object.name": "NUMAPagesTopology", "nova_object.namespace": "nova", "nova_object.version": "1.1", "nova_object.data": {"size_kb": 1048576, "total": 55, "used": 0, "reserved": 8}, "nova_object.changes": ["reserved", "size_kb", "total", "used"]}], "network_metadata": {"nova_object.name": "NetworkMetadata", "nova_object.namespace": "nova", "nova_object.version": "1.0", "nova_object.data": {"physnets": ["datacentre", "dpdk"], "tunneled": true}, "nova_object.changes": ["physnets", "tunneled"]}, "socket": 1}, "nova_object.changes": ["network_metadata", "siblings", "mempages", "memory", "memory_usage", "cpuset", "pinned_cpus", "socket", "id", "pcpuset", "cpu_usage"]}]}, "nova_object.changes": ["cells"]}',pci_device_pools=PciDevicePoolList,ram_allocation_ratio=1.0,running_vms=0,service_id=None,stats={failed_builds='0'},supported_hv_specs=[HVSpec,HVSpec,HVSpec,HVSpec],updated_at=2023-09-21T06:46:30Z,uuid=0458f9f7-8e7c-46ee-8018-dbd71e54ccc7,vcpus=4,vcpus_used=0) _locked_update /usr/lib/python3.9/site-packages/nova/scheduler/host_manager.py:171
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:06.686 14 DEBUG nova.scheduler.host_manager [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Update host state with aggregates: [Aggregate(created_at=2023-09-20T12:09:48Z,deleted=False,deleted_at=None,hosts=['wallaby-cnfv0.example.com','wallaby-cnfv1.example.com'],id=2,metadata={availability_zone='nfv',compute_dpdk='true',trait:COMPUTE_SOCKET_PCI_NUMA_AFFINITY='required'},name='dpdk',updated_at=2023-09-21T06:46:34Z,uuid=f365adfc-92f9-4367-8c6f-a3df69ebec57), Aggregate(created_at=2023-09-20T14:29:18Z,deleted=False,deleted_at=None,hosts=['wallaby-cnfv0.example.com','wallaby-cnfv1.example.com'],id=5,metadata={compute_sriov='true'},name='sriov',updated_at=None,uuid=1a340915-869d-4eaf-a7f8-c5f7ed46dbeb)] _locked_update /usr/lib/python3.9/site-packages/nova/scheduler/host_manager.py:174
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:07.038 14 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Attempting to fit instance cell InstanceNUMACell(cpu_pinning_raw=None,cpu_policy='dedicated',cpu_thread_policy=None,cpu_topology=<?>,cpuset=set([]),cpuset_reserved=None,id=0,memory=8192,pagesize=-2,pcpuset=set([0,1,2,3])) on host_cell NUMACell(cpu_usage=0,cpuset=set([1,25]),id=1,memory=64464,memory_usage=0,mempages=[NUMAPagesTopology,NUMAPagesTopology,NUMAPagesTopology],network_metadata=NetworkMetadata,pcpuset=set([7,9,11,13,15,17,19,21,23,31,33,35,37,39,41,43,45,47]),pinned_cpus=set([]),siblings=[set([13,37]),set([23,47]),set([7,31]),set([9,33]),set([17,41]),set([19,43]),set([1,25]),set([11,35]),set([15,39]),set([21,45])],socket=1) _numa_fit_instance_cell /usr/lib/python3.9/site-packages/nova/virt/hardware.py:929
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:07.038 14 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Selected memory pagesize: 1048576 kB. Requested memory pagesize: -2 (small = -1, large = -2, any = -3) _numa_fit_instance_cell /usr/lib/python3.9/site-packages/nova/virt/hardware.py:943
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:07.038 14 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Instance has requested pinned CPUs _numa_fit_instance_cell /usr/lib/python3.9/site-packages/nova/virt/hardware.py:1021
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:07.039 14 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Packing an instance onto a set of siblings:     host_cell_free_siblings: [{37, 13}, {47, 23}, {31, 7}, {9, 33}, {17, 41}, {43, 19}, set(), {35, 11}, {39, 15}, {45, 21}]    instance_cell: InstanceNUMACell(cpu_pinning_raw=None,cpu_policy='dedicated',cpu_thread_policy=None,cpu_topology=<?>,cpuset=set([]),cpuset_reserved=None,id=0,memory=8192,pagesize=1048576,pcpuset=set([0,1,2,3]))    host_cell_id: 1    threads_per_core: 2    num_cpu_reserved: 0 _pack_instance_onto_cores /usr/lib/python3.9/site-packages/nova/virt/hardware.py:658
/var/log/containers/nova/nova-scheduler.log:2023-09-21 06:47:07.039 14 INFO nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Computed NUMA topology CPU pinning: usable pCPUs: [[37, 13], [47, 23], [31, 7], [9, 33], [17, 41], [43, 19], [35, 11], [39, 15], [45, 21]], vCPUs mapping: [(0, 37), (1, 13), (2, 47), (3, 23)]
wallaby-ctrl0 | FAILED | rc=1 >>
non-zero return code
wallaby-cnfv1 | FAILED | rc=1 >>
non-zero return code
wallaby-cnfv0 | CHANGED | rc=0 >>
/var/log/containers/nova/nova-compute.log:2023-09-21 06:47:07.328 2 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Attempting to fit instance cell InstanceNUMACell(cpu_pinning_raw=None,cpu_policy='dedicated',cpu_thread_policy=None,cpu_topology=<?>,cpuset=set([]),cpuset_reserved=None,id=0,memory=8192,pagesize=-2,pcpuset=set([0,1,2,3])) on host_cell NUMACell(cpu_usage=0,cpuset=set([0,24]),id=0,memory=63703,memory_usage=0,mempages=[NUMAPagesTopology,NUMAPagesTopology,NUMAPagesTopology],network_metadata=NetworkMetadata,pcpuset=set([2,4,6,8,10,12,14,16,18,20,26,28,30,32,34,36,38,40,42,44]),pinned_cpus=set([]),siblings=[set([10,34]),set([12,36]),set([18,42]),set([2,26]),set([6,30]),set([4,28]),set([8,32]),set([16,40]),set([20,44]),set([0,24]),set([38,14])],socket=0) _numa_fit_instance_cell /usr/lib/python3.9/site-packages/nova/virt/hardware.py:929
/var/log/containers/nova/nova-compute.log:2023-09-21 06:47:07.329 2 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Selected memory pagesize: 1048576 kB. Requested memory pagesize: -2 (small = -1, large = -2, any = -3) _numa_fit_instance_cell /usr/lib/python3.9/site-packages/nova/virt/hardware.py:943
/var/log/containers/nova/nova-compute.log:2023-09-21 06:47:07.329 2 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Instance has requested pinned CPUs _numa_fit_instance_cell /usr/lib/python3.9/site-packages/nova/virt/hardware.py:1021
/var/log/containers/nova/nova-compute.log:2023-09-21 06:47:07.329 2 DEBUG nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Packing an instance onto a set of siblings:     host_cell_free_siblings: [{10, 34}, {12, 36}, {18, 42}, {2, 26}, {6, 30}, {4, 28}, {8, 32}, {16, 40}, {20, 44}, set(), {38, 14}]    instance_cell: InstanceNUMACell(cpu_pinning_raw=None,cpu_policy='dedicated',cpu_thread_policy=None,cpu_topology=<?>,cpuset=set([]),cpuset_reserved=None,id=0,memory=8192,pagesize=1048576,pcpuset=set([0,1,2,3]))    host_cell_id: 0    threads_per_core: 2    num_cpu_reserved: 0 _pack_instance_onto_cores /usr/lib/python3.9/site-packages/nova/virt/hardware.py:658
/var/log/containers/nova/nova-compute.log:2023-09-21 06:47:07.331 2 INFO nova.virt.hardware [req-60fb13c3-24b6-45c4-82bc-0298d27e17cf 9af7c9fa0b8346299d2fa8bf5d322489 0b213a7ab3374a90a2d96f38198b4ce2 - default default] Computed NUMA topology CPU pinning: usable pCPUs: [[10, 34], [12, 36], [18, 42], [2, 26], [6, 30], [4, 28], [8, 32], [16, 40], [20, 44], [38, 14]], vCPUs mapping: [(0, 10), (1, 34), (2, 12), (3, 36)]

The OSP controller wallaby-ctrl1 reports:

vCPUs mapping: [(0, 37), (1, 13), (2, 47), (3, 23)]

The OSP compute wallaby-cnfv0 reports:

vCPUs mapping: [(0, 10), (1, 34), (2, 12), (3, 36)]


The expected/correct vCPUs mapping is that of the controller.

[lab] root@wallaby-cnfv0:~# podman exec -it nova_virtqemud bash
[root@wallaby-cnfv0 /]# virsh list
 Id   Name         State
----------------------------
 1    dpdk-vm-20   running

[root@wallaby-cnfv0 /]# virsh vcpupin 1
 VCPU   CPU Affinity
----------------------
 0      10
 1      34
 2      12
 3      36