pci/bus: let it has higher migration priority
PATCH v2 1/1 pci/bus: let it has higher migration priority
In the past, we prioritized IOMMU migration so that we have such a priority order:
IOMMU > PCI Devices
When migrating a guest with both vIOMMU and a pcie-root-port, we'll always migrate vIOMMU first, since pci buses will be seen to have the same priority of general PCI devices.
That's problematic.
The thing is that PCI bus number information is stored in the root port, and that is needed by vIOMMU during post_load(), e.g., to figure out context entry for a device. If we don't have correct bus numbers for devices, we won't be able to recover device state of the DMAR memory regions, and things will be messed up.
So let's boost the PCIe root ports to be even with higher priority:
PCIe Root Port > IOMMU > PCI Devices
A smoke test shows that this patch fixes bug 1538953.
Also, apply this rule to all the PCI bus/bridge devices: ioh3420, xio3130_downstream, xio3130_upstream, pcie_pci_bridge, pci-pci bridge, i82801b11.
I noted that we set pcie_pci_bridge_dev_vmstate twice. Clean that up together.
---
v2:
- add more devices that Marcel mentioned
- rename to MIG_PRI_PCI_BUS
- remove one useless line in existing code
--- hw/pci-bridge/gen_pcie_root_port.c | 1 + hw/pci-bridge/i82801b11.c | 1 + hw/pci-bridge/ioh3420.c | 1 + hw/pci-bridge/pci_bridge_dev.c | 1 + hw/pci-bridge/pcie_pci_bridge.c | 2 +- hw/pci-bridge/xio3130_downstream.c | 1 + hw/pci-bridge/xio3130_upstream.c | 1 + include/migration/vmstate.h | 1 + 8 files changed, 8 insertions(+), 1 deletion(-)
hw/pci-bridge/gen_pcie_root_port.c +1 -0
@@ -101,6 +101,7 @@ static void gen_rp_realize(DeviceState *dev, Error **errp)static const VMStateDescription vmstate_rp_dev = { .name ="pcie-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,
hw/pci-bridge/i82801b11.c +1 -0
@@ -80,6 +80,7 @@ err_bridge:static const VMStateDescription i82801b11_bridge_dev_vmstate = { .name ="i82801b11_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),VMSTATE_END_OF_LIST ()
hw/pci-bridge/ioh3420.c +1 -0
@@ -83,6 +83,7 @@ static void ioh3420_interrupts_uninit(PCIDevice *d)static const VMStateDescription vmstate_ioh3420 = { .name ="ioh-3240-express-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,
hw/pci-bridge/pci_bridge_dev.c +1 -0
@@ -174,6 +174,7 @@ static bool pci_device_shpc_present(void *opaque, int version_id)static const VMStateDescription pci_bridge_dev_vmstate = { .name ="pci_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice, pci_device_shpc_present),
hw/pci-bridge/pcie_pci_bridge.c +1 -1
@@ -129,6 +129,7 @@ static Property pcie_pci_bridge_dev_properties[] = {static const VMStateDescription pcie_pci_bridge_dev_vmstate = { .name =TYPE_PCIE_PCI_BRIDGE_DEV , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice,NULL ),
@@ -178,7 +179,6 @@ static void pcie_pci_bridge_class_init(ObjectClass *klass, void *data) k->config_write = pcie_pci_bridge_write_config; dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->props = pcie_pci_bridge_dev_properties; - dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->reset = &pcie_pci_bridge_reset;set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); hc->plug = pcie_pci_bridge_hotplug_cb;
hw/pci-bridge/xio3130_downstream.c +1 -0
@@ -161,6 +161,7 @@ static Property xio3130_downstream_props[] = {static const VMStateDescription vmstate_xio3130_downstream = { .name ="xio3130-express-downstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,
hw/pci-bridge/xio3130_upstream.c +1 -0
@@ -133,6 +133,7 @@ PCIEPort *xio3130_upstream_init(PCIBus *bus, int devfn, bool multifunction,static const VMStateDescription vmstate_xio3130_upstream = { .name ="xio3130-express-upstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .fields = (VMStateField []) {
include/migration/vmstate.h +1 -0
@@ -148,6 +148,7 @@ enum VMStateFlags { typedef enum { MIG_PRI_DEFAULT = 0, MIG_PRI_IOMMU, /* Must happen before PCI devices */ + MIG_PRI_PCI_BUS, /* Must happen before IOMMU */ MIG_PRI_GICV3_ITS, /* Must happen before PCI devices */ MIG_PRI_GICV3, /* Must happen before the ITS */ MIG_PRI_MAX,
Signature · 1 line
2.14.3
Quoted 153 lines · Peter Xu
In the past, we prioritized IOMMU migration so that we have such a priority order:
IOMMU > PCI Devices
When migrating a guest with both vIOMMU and a pcie-root-port, we'll always migrate vIOMMU first, since pci buses will be seen to have the same priority of general PCI devices.
That's problematic.
The thing is that PCI bus number information is stored in the root port, and that is needed by vIOMMU during
post_load(), e.g., to figure out context entry for a device. If we don't have correct bus numbers for devices, we won't be able to recover device state of the DMAR memory regions, and things will be messed up.So let's boost the PCIe root ports to be even with higher priority:
PCIe Root Port > IOMMU > PCI Devices
A smoke test shows that this patch fixes bug 1538953.
Also, apply this rule to all the PCI bus/bridge devices: ioh3420,
xio3130_downstream,xio3130_upstream,pcie_pci_bridge, pci-pci bridge, i82801b11.I noted that we set
pcie_pci_bridge_dev_vmstatetwice. Clean that up together.CC Alex Williamson <[email protected]>CC Marcel Apfelbaum <[email protected]>CC Michael S. Tsirkin <[email protected]>CC Dr. David Alan Gilbert <[email protected]>CC Juan Quintela <[email protected]>CC Laurent Vivier <[email protected]>Reported-by Maxime Coquelin <[email protected]>Signed-off-by Peter Xu <[email protected]>---
v2:
- add more devices that Marcel mentioned
- rename toMIG_PRI_PCI_BUS
- remove one useless line in existing code--- hw/pci-bridge/gen_pcie_root_port.c | 1 + hw/pci-bridge/i82801b11.c | 1 + hw/pci-bridge/ioh3420.c | 1 + hw/pci-bridge/pci_bridge_dev.c | 1 + hw/pci-bridge/pcie_pci_bridge.c | 2 +- hw/pci-bridge/xio3130_downstream.c | 1 + hw/pci-bridge/xio3130_upstream.c | 1 + include/migration/vmstate.h | 1 + 8 files changed, 8 insertions(+), 1 deletion(-)hw/pci-bridge/gen_pcie_root_port.c +1
-0@@ -101,6 +101,7 @@ static void gen_rp_realize(DeviceState *dev, Error **errp)static const VMStateDescription vmstate_rp_dev = { .name ="pcie-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/i82801b11.c +1
-0@@ -80,6 +80,7 @@ err_bridge:static const VMStateDescription i82801b11_bridge_dev_vmstate = { .name ="i82801b11_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),VMSTATE_END_OF_LIST ()hw/pci-bridge/ioh3420.c +1
-0@@ -83,6 +83,7 @@ static void ioh3420_interrupts_uninit(PCIDevice *d)static const VMStateDescription vmstate_ioh3420 = { .name ="ioh-3240-express-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/pci_bridge_dev.c +1
-0@@ -174,6 +174,7 @@ static bool pci_device_shpc_present(void *opaque, int version_id)static const VMStateDescription pci_bridge_dev_vmstate = { .name ="pci_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice, pci_device_shpc_present),hw/pci-bridge/pcie_pci_bridge.c +1
-1@@ -129,6 +129,7 @@ static Property pcie_pci_bridge_dev_properties[] = {static const VMStateDescription pcie_pci_bridge_dev_vmstate = { .name =TYPE_PCIE_PCI_BRIDGE_DEV , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice,NULL ),@@ -178,7 +179,6 @@ static void pcie_pci_bridge_class_init(ObjectClass *klass, void *data) k->config_write = pcie_pci_bridge_write_config; dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->props = pcie_pci_bridge_dev_properties; - dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->reset = &pcie_pci_bridge_reset;set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); hc->plug = pcie_pci_bridge_hotplug_cb;hw/pci-bridge/xio3130_downstream.c +1
-0@@ -161,6 +161,7 @@ static Property xio3130_downstream_props[] = {static const VMStateDescription vmstate_xio3130_downstream = { .name ="xio3130-express-downstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/xio3130_upstream.c +1
-0@@ -133,6 +133,7 @@ PCIEPort *xio3130_upstream_init(PCIBus *bus, int devfn, bool multifunction,static const VMStateDescription vmstate_xio3130_upstream = { .name ="xio3130-express-upstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .fields = (VMStateField []) {include/migration/vmstate.h +0
-0
include/migration/vmstate.h enum VMStateFlags { · line 154
Quoted 7 lines
include/migration/vmstate.h +1
-0@@ -148,6 +148,7 @@ enum VMStateFlags { typedef enum { MIG_PRI_DEFAULT = 0, MIG_PRI_IOMMU, /* Must happen before PCI devices */ + MIG_PRI_PCI_BUS, /* Must happen before IOMMU */ MIG_PRI_GICV3_ITS, /* Must happen before PCI devices */ MIG_PRI_GICV3, /* Must happen before the ITS */ MIG_PRI_MAX,
Thanks,
Marcel
Quoted 30 lines · Peter Xu
In the past, we prioritized IOMMU migration so that we have such a priority order:
IOMMU > PCI Devices
When migrating a guest with both vIOMMU and a pcie-root-port, we'll always migrate vIOMMU first, since pci buses will be seen to have the same priority of general PCI devices.
That's problematic.
The thing is that PCI bus number information is stored in the root port, and that is needed by vIOMMU during
post_load(), e.g., to figure out context entry for a device. If we don't have correct bus numbers for devices, we won't be able to recover device state of the DMAR memory regions, and things will be messed up.So let's boost the PCIe root ports to be even with higher priority:
PCIe Root Port > IOMMU > PCI Devices
A smoke test shows that this patch fixes bug 1538953.
Also, apply this rule to all the PCI bus/bridge devices: ioh3420,
xio3130_downstream,xio3130_upstream,pcie_pci_bridge, pci-pci bridge, i82801b11.I noted that we set
pcie_pci_bridge_dev_vmstatetwice. Clean that up together.
Quoted 9 lines
CC Alex Williamson <[email protected]>CC Marcel Apfelbaum <[email protected]>CC Michael S. Tsirkin <[email protected]>CC Dr. David Alan Gilbert <[email protected]>CC Juan Quintela <[email protected]>CC Laurent Vivier <[email protected]>Reported-by Maxime Coquelin <[email protected]>Signed-off-by Peter Xu <[email protected]>
dgilbert, could you pls confirm this looks like a sane approach to you?
Quoted 122 lines
---
v2:
- add more devices that Marcel mentioned
- rename toMIG_PRI_PCI_BUS
- remove one useless line in existing code--- hw/pci-bridge/gen_pcie_root_port.c | 1 + hw/pci-bridge/i82801b11.c | 1 + hw/pci-bridge/ioh3420.c | 1 + hw/pci-bridge/pci_bridge_dev.c | 1 + hw/pci-bridge/pcie_pci_bridge.c | 2 +- hw/pci-bridge/xio3130_downstream.c | 1 + hw/pci-bridge/xio3130_upstream.c | 1 + include/migration/vmstate.h | 1 + 8 files changed, 8 insertions(+), 1 deletion(-)hw/pci-bridge/gen_pcie_root_port.c +1
-0@@ -101,6 +101,7 @@ static void gen_rp_realize(DeviceState *dev, Error **errp)static const VMStateDescription vmstate_rp_dev = { .name ="pcie-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/i82801b11.c +1
-0@@ -80,6 +80,7 @@ err_bridge:static const VMStateDescription i82801b11_bridge_dev_vmstate = { .name ="i82801b11_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),VMSTATE_END_OF_LIST ()hw/pci-bridge/ioh3420.c +1
-0@@ -83,6 +83,7 @@ static void ioh3420_interrupts_uninit(PCIDevice *d)static const VMStateDescription vmstate_ioh3420 = { .name ="ioh-3240-express-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/pci_bridge_dev.c +1
-0@@ -174,6 +174,7 @@ static bool pci_device_shpc_present(void *opaque, int version_id)static const VMStateDescription pci_bridge_dev_vmstate = { .name ="pci_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice, pci_device_shpc_present),hw/pci-bridge/pcie_pci_bridge.c +1
-1@@ -129,6 +129,7 @@ static Property pcie_pci_bridge_dev_properties[] = {static const VMStateDescription pcie_pci_bridge_dev_vmstate = { .name =TYPE_PCIE_PCI_BRIDGE_DEV , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice,NULL ),@@ -178,7 +179,6 @@ static void pcie_pci_bridge_class_init(ObjectClass *klass, void *data) k->config_write = pcie_pci_bridge_write_config; dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->props = pcie_pci_bridge_dev_properties; - dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->reset = &pcie_pci_bridge_reset;set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); hc->plug = pcie_pci_bridge_hotplug_cb;hw/pci-bridge/xio3130_downstream.c +1
-0@@ -161,6 +161,7 @@ static Property xio3130_downstream_props[] = {static const VMStateDescription vmstate_xio3130_downstream = { .name ="xio3130-express-downstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/xio3130_upstream.c +1
-0@@ -133,6 +133,7 @@ PCIEPort *xio3130_upstream_init(PCIBus *bus, int devfn, bool multifunction,static const VMStateDescription vmstate_xio3130_upstream = { .name ="xio3130-express-upstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .fields = (VMStateField []) {include/migration/vmstate.h +1
-0@@ -148,6 +148,7 @@ enum VMStateFlags { typedef enum { MIG_PRI_DEFAULT = 0, MIG_PRI_IOMMU, /* Must happen before PCI devices */ + MIG_PRI_PCI_BUS, /* Must happen before IOMMU */ MIG_PRI_GICV3_ITS, /* Must happen before PCI devices */ MIG_PRI_GICV3, /* Must happen before the ITS */ MIG_PRI_MAX,Signature · 1 line
2.14.3
Quoted 41 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 30 lines · Peter Xu
In the past, we prioritized IOMMU migration so that we have such a priority order:
IOMMU > PCI Devices
When migrating a guest with both vIOMMU and a pcie-root-port, we'll always migrate vIOMMU first, since pci buses will be seen to have the same priority of general PCI devices.
That's problematic.
The thing is that PCI bus number information is stored in the root port, and that is needed by vIOMMU during
post_load(), e.g., to figure out context entry for a device. If we don't have correct bus numbers for devices, we won't be able to recover device state of the DMAR memory regions, and things will be messed up.So let's boost the PCIe root ports to be even with higher priority:
PCIe Root Port > IOMMU > PCI Devices
A smoke test shows that this patch fixes bug 1538953.
Also, apply this rule to all the PCI bus/bridge devices: ioh3420,
xio3130_downstream,xio3130_upstream,pcie_pci_bridge, pci-pci bridge, i82801b11.I noted that we set
pcie_pci_bridge_dev_vmstatetwice. Clean that up together.Quoted 9 lines
CC Alex Williamson <[email protected]>CC Marcel Apfelbaum <[email protected]>CC Michael S. Tsirkin <[email protected]>CC Dr. David Alan Gilbert <[email protected]>CC Juan Quintela <[email protected]>CC Laurent Vivier <[email protected]>Reported-by Maxime Coquelin <[email protected]>Signed-off-by Peter Xu <[email protected]>
Quoted 1 line
dgilbert, could you pls confirm this looks like a sane approach to you?
Yes it does, although a couple of (minor) thoughts:
a) I'm a little confused whether this needs to apply to all PCI
busses (and hence why it's MIG_PRI_PCI_BUS) or only the root ports
b) It's probably worth getting one of the Arm [e[;e tp check the
order relative to the GICV3 entries; cc'ing Eric.
Dave
Quoted 114 lines
Quoted 114 lines
---
v2:
- add more devices that Marcel mentioned
- rename toMIG_PRI_PCI_BUS
- remove one useless line in existing code--- hw/pci-bridge/gen_pcie_root_port.c | 1 + hw/pci-bridge/i82801b11.c | 1 + hw/pci-bridge/ioh3420.c | 1 + hw/pci-bridge/pci_bridge_dev.c | 1 + hw/pci-bridge/pcie_pci_bridge.c | 2 +- hw/pci-bridge/xio3130_downstream.c | 1 + hw/pci-bridge/xio3130_upstream.c | 1 + include/migration/vmstate.h | 1 + 8 files changed, 8 insertions(+), 1 deletion(-)hw/pci-bridge/gen_pcie_root_port.c +1
-0@@ -101,6 +101,7 @@ static void gen_rp_realize(DeviceState *dev, Error **errp)static const VMStateDescription vmstate_rp_dev = { .name ="pcie-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/i82801b11.c +1
-0@@ -80,6 +80,7 @@ err_bridge:static const VMStateDescription i82801b11_bridge_dev_vmstate = { .name ="i82801b11_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),VMSTATE_END_OF_LIST ()hw/pci-bridge/ioh3420.c +1
-0@@ -83,6 +83,7 @@ static void ioh3420_interrupts_uninit(PCIDevice *d)static const VMStateDescription vmstate_ioh3420 = { .name ="ioh-3240-express-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/pci_bridge_dev.c +1
-0@@ -174,6 +174,7 @@ static bool pci_device_shpc_present(void *opaque, int version_id)static const VMStateDescription pci_bridge_dev_vmstate = { .name ="pci_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice, pci_device_shpc_present),hw/pci-bridge/pcie_pci_bridge.c +1
-1@@ -129,6 +129,7 @@ static Property pcie_pci_bridge_dev_properties[] = {static const VMStateDescription pcie_pci_bridge_dev_vmstate = { .name =TYPE_PCIE_PCI_BRIDGE_DEV , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice,NULL ),@@ -178,7 +179,6 @@ static void pcie_pci_bridge_class_init(ObjectClass *klass, void *data) k->config_write = pcie_pci_bridge_write_config; dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->props = pcie_pci_bridge_dev_properties; - dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->reset = &pcie_pci_bridge_reset;set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); hc->plug = pcie_pci_bridge_hotplug_cb;hw/pci-bridge/xio3130_downstream.c +1
-0@@ -161,6 +161,7 @@ static Property xio3130_downstream_props[] = {static const VMStateDescription vmstate_xio3130_downstream = { .name ="xio3130-express-downstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/xio3130_upstream.c +1
-0@@ -133,6 +133,7 @@ PCIEPort *xio3130_upstream_init(PCIBus *bus, int devfn, bool multifunction,static const VMStateDescription vmstate_xio3130_upstream = { .name ="xio3130-express-upstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .fields = (VMStateField []) {include/migration/vmstate.h +0
-0@@ -148,6 +148,7 @@ enum VMStateFlags { typedef enum {
Quoted 8 lines
Quoted 8 lines
include/migration/vmstate.h +1
-0MIG_PRI_DEFAULT =0 ,MIG_PRI_IOMMU ,/* Must happen before PCI devices */ +MIG_PRI_PCI_BUS ,/* Must happen before IOMMU */ MIG_PRI_GICV3_ITS ,/* Must happen before PCI devices */ MIG_PRI_GICV3 ,/* Must happen before the ITS */ MIG_PRI_MAX,Signature · 1 line
2.14.3
--
Dr. David Alan Gilbert / [email protected] / Manchester, UK
Hi,
Quoted 42 lines · Dr. David Alan Gilbert
Quoted 41 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 39 lines · Peter Xu
In the past, we prioritized IOMMU migration so that we have such a priority order:
IOMMU > PCI Devices
When migrating a guest with both vIOMMU and a pcie-root-port, we'll always migrate vIOMMU first, since pci buses will be seen to have the same priority of general PCI devices.
That's problematic.
The thing is that PCI bus number information is stored in the root port, and that is needed by vIOMMU during
post_load(), e.g., to figure out context entry for a device. If we don't have correct bus numbers for devices, we won't be able to recover device state of the DMAR memory regions, and things will be messed up.So let's boost the PCIe root ports to be even with higher priority:
PCIe Root Port > IOMMU > PCI Devices
A smoke test shows that this patch fixes bug 1538953.
Also, apply this rule to all the PCI bus/bridge devices: ioh3420,
xio3130_downstream,xio3130_upstream,pcie_pci_bridge, pci-pci bridge, i82801b11.I noted that we set
pcie_pci_bridge_dev_vmstatetwice. Clean that up together.CC Alex Williamson <[email protected]>CC Marcel Apfelbaum <[email protected]>CC Michael S. Tsirkin <[email protected]>CC Dr. David Alan Gilbert <[email protected]>CC Juan Quintela <[email protected]>CC Laurent Vivier <[email protected]>Reported-by Maxime Coquelin <[email protected]>Signed-off-by Peter Xu <[email protected]>
Quoted 7 lines
Quoted 1 line
dgilbert, could you pls confirm this looks like a sane approach to you?
Yes it does, although a couple of (minor) thoughts:
a) I'm a little confused whether this needs to apply to all PCI
busses (and hence why it'sMIG_PRI_PCI_BUS) or only the root ports
b) It's probably worth getting one of the Arm [e[;e tp check the
order relative to the GICV3 entries; cc'ing Eric.
The order looks OK to me. My understanding is all devices likely to generate MSIs must be restored after the ITS MSI controller which is the case here.
Thanks
Eric
Quoted 129 lines
Dave
Quoted 114 lines
Quoted 114 lines
---
v2:
- add more devices that Marcel mentioned
- rename toMIG_PRI_PCI_BUS
- remove one useless line in existing code--- hw/pci-bridge/gen_pcie_root_port.c | 1 + hw/pci-bridge/i82801b11.c | 1 + hw/pci-bridge/ioh3420.c | 1 + hw/pci-bridge/pci_bridge_dev.c | 1 + hw/pci-bridge/pcie_pci_bridge.c | 2 +- hw/pci-bridge/xio3130_downstream.c | 1 + hw/pci-bridge/xio3130_upstream.c | 1 + include/migration/vmstate.h | 1 + 8 files changed, 8 insertions(+), 1 deletion(-)hw/pci-bridge/gen_pcie_root_port.c +1
-0@@ -101,6 +101,7 @@ static void gen_rp_realize(DeviceState *dev, Error **errp)static const VMStateDescription vmstate_rp_dev = { .name ="pcie-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/i82801b11.c +1
-0@@ -80,6 +80,7 @@ err_bridge:static const VMStateDescription i82801b11_bridge_dev_vmstate = { .name ="i82801b11_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),VMSTATE_END_OF_LIST ()hw/pci-bridge/ioh3420.c +1
-0@@ -83,6 +83,7 @@ static void ioh3420_interrupts_uninit(PCIDevice *d)static const VMStateDescription vmstate_ioh3420 = { .name ="ioh-3240-express-root-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/pci_bridge_dev.c +1
-0@@ -174,6 +174,7 @@ static bool pci_device_shpc_present(void *opaque, int version_id)static const VMStateDescription pci_bridge_dev_vmstate = { .name ="pci_bridge" , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice, pci_device_shpc_present),hw/pci-bridge/pcie_pci_bridge.c +1
-1@@ -129,6 +129,7 @@ static Property pcie_pci_bridge_dev_properties[] = {static const VMStateDescription pcie_pci_bridge_dev_vmstate = { .name =TYPE_PCIE_PCI_BRIDGE_DEV , + .priority =MIG_PRI_PCI_BUS , .fields = (VMStateField []) {VMSTATE_PCI_DEVICE (parent_obj, PCIBridge),SHPC_VMSTATE (shpc, PCIDevice,NULL ),@@ -178,7 +179,6 @@ static void pcie_pci_bridge_class_init(ObjectClass *klass, void *data) k->config_write = pcie_pci_bridge_write_config; dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->props = pcie_pci_bridge_dev_properties; - dc->vmsd = &pcie_pci_bridge_dev_vmstate; dc->reset = &pcie_pci_bridge_reset;set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); hc->plug = pcie_pci_bridge_hotplug_cb;hw/pci-bridge/xio3130_downstream.c +1
-0@@ -161,6 +161,7 @@ static Property xio3130_downstream_props[] = {static const VMStateDescription vmstate_xio3130_downstream = { .name ="xio3130-express-downstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .post_load = pcie_cap_slot_post_load,hw/pci-bridge/xio3130_upstream.c +1
-0@@ -133,6 +133,7 @@ PCIEPort *xio3130_upstream_init(PCIBus *bus, int devfn, bool multifunction,static const VMStateDescription vmstate_xio3130_upstream = { .name ="xio3130-express-upstream-port" , + .priority =MIG_PRI_PCI_BUS , .version_id =1 , .minimum_version_id =1 , .fields = (VMStateField []) {include/migration/vmstate.h +0
-0@@ -148,6 +148,7 @@ enum VMStateFlags { typedef enum {Quoted 8 lines
Quoted 8 lines
include/migration/vmstate.h +1
-0MIG_PRI_DEFAULT =0 ,MIG_PRI_IOMMU ,/* Must happen before PCI devices */ +MIG_PRI_PCI_BUS ,/* Must happen before IOMMU */ MIG_PRI_GICV3_ITS ,/* Must happen before PCI devices */ MIG_PRI_GICV3 ,/* Must happen before the ITS */ MIG_PRI_MAX,Signature · 1 line
2.14.3--
Dr. David Alan Gilbert / [email protected] / Manchester, UK