NVMe namespace hotplug and drive reconnection support
cover NVMe namespace hotplug and drive reconnection support
This series adds two features that together enable transparent NVMe disk hot-swap simulation in QEMU, matching the behavior of physical NVMe drives being pulled and reinserted in the same PCIe slot.
Problem:
Currently, hot-swapping an NVMe disk in QEMU requires removing the entire NVMe controller via device_del, which causes the Linux guest to assign a new controller number on re-add (e.g. nvme2 becomes nvme4). This breaks storage software that tracks drives by device name.
Solution:
Patch 1 adds hotplug support for nvme-ns devices on the NvmeBus, with proper Asynchronous Event Notification (AEN) so the guest kernel detects namespace changes. This allows namespace-level hot-swap without removing the NVMe controller.
Patch 2 adds a drive_insert HMP command that reconnects a host block device file to an existing guest device after drive_del. This is the counterpart to drive_del for non-removable devices where blockdev-change-medium cannot be used.
The recommended hot-swap sequence is:
1. drive_del <drive-id> # disconnect backing store
2. drive_insert <device> <file> # reconnect backing store
3. pcie_aer_inject_error <port> SDN # trigger controller reset
After this sequence, the guest sees the same controller and namespace names (e.g. /dev/nvme2n1 remains /dev/nvme2n1), and the NVMe driver recovers transparently via the standard AER recovery path.
Tested with:
- Linux 6.1 guest on QEMU aarch64 with HVF (macOS)
- NVMe subsystem model with multipath disabled
- DirectPV and MinIO AIStor storage stack
mr-083 (2):
hw/nvme: add namespace hotplug support
block/monitor: add drive_insert HMP command
block/monitor/block-hmp-cmds.c | 59 +++++++++++++++++++++++ hmp-commands.hx | 18 +++++++ hw/nvme/ctrl.c | 85 ++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 1 + hw/nvme/subsys.c | 2 + include/block/block-hmp-cmds.h | 1 + 6 files changed, 166 insertions(+)
--
2.50.1 (Apple Git-155)
PATCH 1/2 hw/nvme: add namespace hotplug support
Add hotplug support for nvme-ns devices on the NvmeBus. This enables namespace-level hot-swap without removing the NVMe controller, which is how physical NVMe drives behave when hot-swapped in the same PCIe slot.
Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with NS_ATTR_CHANGED so the guest kernel rescans namespaces
- unplug: detach from all controllers, send AEN, remove from subsystem, then unrealize the device
The plug handler skips controllers that haven't started yet (qs_created == false) to avoid interfering with boot-time namespace attachment in nvme_start_ctrl().
Both the controller bus and subsystem bus are configured as hotplug handlers via qbus_set_bus_hotplug_handler() since nvme-ns devices may reparent to the subsystem bus during realize.
--- hw/nvme/ctrl.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 1 + hw/nvme/subsys.c | 2 ++ 3 files changed, 88 insertions(+)
hw/nvme/ctrl.c +85 -0
@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h"
@@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return ;
@@ -9553,10 +9555,93 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Detach from all controllers and notify the guest via AEN. +* Must happen before unrealize to avoid use-after-free when the +* guest sends I/O to a freed namespace. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL ); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* Remove from subsystem namespace list. */ + subsys->namespaces[nsid] =NULL ; + +/* +* Unrealize: drain I/O, flush, cleanup structures, remove from QOM. +* nvme_ns_unrealize() handles drain/shutdown/cleanup internally. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )
hw/nvme/ns.c +1 -0
@@ -937,6 +937,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type =TYPE_NVME_BUS ; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ; }
hw/nvme/subsys.c +2 -0
@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h"
@@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp)NvmeSubsystem *subsys =NVME_SUBSYS (dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp); }
Signature · 1 line
2.50.1 (Apple Git-155)
PATCH 2/2 block/monitor: add drive_insert HMP command
In reply to mr-083Add a drive_insert HMP command that reconnects a host block device file to an existing guest device whose backing store was previously removed with drive_del.
After drive_del, the BlockBackend remains attached to the guest device but has no BlockDriverState (shown as "[not inserted]" in info block). drive_insert opens the specified file, finds the device's BlockBackend by iterating all backends and matching the attached device ID, then calls blk_insert_bs() to reconnect the backing store.
This complements drive_del for non-removable devices (such as NVMe namespaces) where blockdev-change-medium cannot be used. Combined with PCIe AER Surprise Down error injection to trigger a controller reset, this enables complete NVMe disk hot-swap simulation where the guest sees the same device names throughout.
Example usage:
drive_del drv0 # remove backing store
drive_insert ns0 disk.qcow2 # reconnect backing pcie_aer_inject_error rp0 SDN # trigger controller reset
--- block/monitor/block-hmp-cmds.c | 59 ++++++++++++++++++++++++++++++++++ hmp-commands.hx | 18 +++++++++++ include/block/block-hmp-cmds.h | 1 + 3 files changed, 78 insertions(+)
block/monitor/block-hmp-cmds.c +59 -0
@@ -38,7 +38,9 @@#include "qemu/osdep.h" #include "hw/core/boards.h" #include "system/block-backend.h" +#include "system/block-backend-global-state.h" #include "system/blockdev.h" +#include "block/block-global-state.h" #include "qapi/qapi-commands-block.h" #include "qapi/qapi-commands-block-export.h" #include "qobject/qdict.h"
@@ -195,6 +197,63 @@ unlock:hmp_handle_error (mon, err); } +void hmp_drive_insert (Monitor *mon,const QDict *qdict) +{ +const char *id =qdict_get_str (qdict,"id" ); +const char *filename =qdict_get_str (qdict,"filename" ); +BlockBackend *blk =NULL ; +BlockBackend *iter; +BlockDriverState *bs; +Error *err =NULL ; + +GLOBAL_STATE_CODE (); + +/* +* After drive_del, the BlockBackend is removed from the monitor name +* registry but still attached to the device. Find it by iterating all +* BlockBackends and matching by the device ID shown in "info block". +*/ +for (iter =blk_all_next (NULL ); iter; iter =blk_all_next (iter)) { +DeviceState *dev =blk_get_attached_dev (iter); +if (dev && dev->id &&strcmp (dev->id, id) ==0 ) { + blk = iter; +break ; + } + } + +if (!blk) { +/* Fallback: try by block backend name */ + blk =blk_by_name (id); + } + +if (!blk) { +error_setg (&err,"Device '%s' not found" , id); + goto out; + } + +if (blk_bs (blk)) { +error_setg (&err,"Device '%s' already has a medium inserted" , id); + goto out; + } + + bs =bdrv_open (filename,NULL ,NULL ,BDRV_O_RDWR , &err); +if (!bs) { + goto out; + } + +if (blk_insert_bs (blk, bs, &err) <0 ) { +bdrv_unref (bs); + goto out; + } + +bdrv_unref (bs); +monitor_printf (mon,"OK\n" ); +return ; + +out: +hmp_handle_error (mon, err); +} +void hmp_commit (Monitor *mon,const QDict *qdict) {const char *device =qdict_get_str (qdict,"device" );
hmp-commands.hx +18 -0
@@ -207,6 +207,24 @@ SRST actions (drive options rerror, werror). ERST + { + .name = "drive_insert", + .args_type = "id:B,filename:F", + .params = "device filename", + .help = "insert a host block device into an empty drive", + .cmd = hmp_drive_insert, + }, + +SRST +``drive_insert`` *device* *filename* + Insert a host block device file into a drive that has been emptied by + ``drive_del``. This reconnects the backing store without removing the + guest device, enabling transparent disk hot-swap for non-removable devices + such as NVMe namespaces. Combined with PCIe AER Surprise Down error + injection (``pcie_aer_inject_error`` *device* ``SDN``), this enables + complete NVMe disk hot-swap simulation. +ERST + { .name = "change", .args_type = "device:B,force:-f,target:F,arg:s?,read-only-mode:s?",
include/block/block-hmp-cmds.h +1 -0
@@ -21,6 +21,7 @@ void hmp_drive_add(Monitor *mon, const QDict *qdict);void hmp_commit (Monitor *mon,const QDict *qdict);void hmp_drive_del (Monitor *mon,const QDict *qdict); +void hmp_drive_insert (Monitor *mon,const QDict *qdict);void hmp_drive_mirror (Monitor *mon,const QDict *qdict);void hmp_drive_backup (Monitor *mon,const QDict *qdict);
Signature · 1 line
2.50.1 (Apple Git-155)
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
Quoted 16 lines · On Thu, Apr 09, 2026 at 08:01:54AM +0200, mr-083 wrote:
Add a
drive_insertHMP command that reconnects a host block device file to an existing guest device whose backing store was previously removed withdrive_del.After
drive_del, the BlockBackend remains attached to the guest device but has no BlockDriverState (shown as "[not inserted]" in info block).drive_insertopens the specified file, finds the device's BlockBackend by iterating all backends and matching the attached device ID, then callsblk_insert_bs()to reconnect the backing store.This complements
drive_delfor non-removable devices (such as NVMe namespaces) where blockdev-change-medium cannot be used. Combined with PCIe AER Surprise Down error injection to trigger a controller reset, this enables complete NVMe disk hot-swap simulation where the guest sees the same device names throughout.
Quoted 4 lines
Example usage:
drive_deldrv0 # remove backing store
drive_insertns0 disk.qcow2 # reconnect backingpcie_aer_inject_errorrp0 SDN # trigger controller reset
This approach does not delete the --device nvme-ns but instead replaces the BlockBackend's root node so that the existing NVMe Namespace changes the underlying storage.
Questions about this approach:
1. NVMe AEN is not involved at all?
2. Does the guest have access to the new storage as soon as drive_insert completes and the PCIe AER is just to kick the guest driver (e.g. getting it to update the size of the guest Linux block device)?
Stefan
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
Hello Stefan,
1. Correct, no AEN involved.The namespace stays attached to the controller throughout. Only the BlockBackend's root node changes, so from the NVMe protocol perspective nothing happened to the namespace topology.
2. Yes, the new storage is accessible immediately after drive_insert at the QEMU block layer level. The SDN is needed to reset the NVMe controller which may have cached an error state from I/O failures during the period when no backing was present. Without the reset, the guest driver continues to see the controller as faulted. The SDN triggers the standard PCIe AER recovery path (freeze → slot reset → restart) which clears the error state and resumes normal I/O.
One thing to note: since the namespace device stays alive, the guest kernel's block device number is preserved (e.g. /dev/nvme0n1 stays nvme0n1). This avoids the ida_alloc renaming issue that occurs with device_del + device_add when filesystems hold references to the old namespace head.
Thanks
.
Matthieu
www.min.io <>
[email protected] <>
Quoted 35 lines · Stefan Hajnoczi
Quoted 16 lines · On Thu, Apr 09, 2026 at 08:01:54AM +0200, mr-083 wrote:
Add a
drive_insertHMP command that reconnects a host block device file to an existing guest device whose backing store was previously removed withdrive_del.After
drive_del, the BlockBackend remains attached to the guest device but has no BlockDriverState (shown as "[not inserted]" in info block).drive_insertopens the specified file, finds the device's BlockBackend by iterating all backends and matching the attached device ID, then callsblk_insert_bs()to reconnect the backing store.This complements
drive_delfor non-removable devices (such as NVMe namespaces) where blockdev-change-medium cannot be used. Combined with PCIe AER Surprise Down error injection to trigger a controller reset, this enables complete NVMe disk hot-swap simulation where the guest sees the same device names throughout.Quoted 4 lines
Example usage:
drive_deldrv0 # remove backing store
drive_insertns0 disk.qcow2 # reconnect backingpcie_aer_inject_errorrp0 SDN # trigger controller resetThis approach does not delete the
--device nvme-nsbut instead replaces the BlockBackend's root node so that the existing NVMe Namespace changes the underlying storage.Questions about this approach:
1. NVMe AEN is not involved at all?
2. Does the guest have access to the new storage as soon asdrive_insertcompletes and the PCIe AER is just to kick the guest driver (e.g. getting it to update the size of the guest Linux block device)?Stefan
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
Quoted 14 lines · Matthieu Rolla
Hello Stefan,
1. Correct, no AEN involved.The namespace stays attached to the controller throughout. Only the BlockBackend's root node changes, so from the NVMe protocol perspective nothing happened to the namespace topology.
2. Yes, the new storage is accessible immediately after
drive_insertat the QEMU block layer level. The SDN is needed to reset the NVMe controller which may have cached an error state from I/O failures during the period when no backing was present. Without the reset, the guest driver continues to see the controller as faulted. The SDN triggers the standard PCIe AER recovery path (freeze → slot reset → restart) which clears the error state and resumes normal I/O.
Quoted 6 lines
One thing to note: since the namespace device stays alive, the guest kernel's block device number is preserved (e.g.
/dev/nvme0n1stays nvme0n1). This avoids theida_allocrenaming issue that occurs withdevice_del+device_addwhen filesystems hold references to the old namespace head.
OK. I had hoped this would let me test my FreeBSD namespace arrival and departure messages correctly. I just added the code for AWS (and other VM) support for attaching / detaching drives and was hoping I could turn this into a regression test, but sounds like no.
I'll have to go look at what the Linux driver does here. Right now Surprise Down events just increment a counter on FreeBSD... Will thta work for a DPC system too? Or is that not relevant for the current state of the qemu art?
Warner
Quoted 49 lines
Thanks
.
*Matthieu*
www.min.io
[email protected]On Apr 14, 2026, at 7:57 PM, Stefan Hajnoczi <[email protected]> wrote:
On Thu, Apr 09, 2026 at 08:01:54AM +0200, mr-083 wrote:
Add a
drive_insertHMP command that reconnects a host block device file to an existing guest device whose backing store was previously removed withdrive_del.After
drive_del, the BlockBackend remains attached to the guest device but has no BlockDriverState (shown as "[not inserted]" in info block).drive_insertopens the specified file, finds the device's BlockBackend by iterating all backends and matching the attached device ID, then callsblk_insert_bs()to reconnect the backing store.This complements
drive_delfor non-removable devices (such as NVMe namespaces) where blockdev-change-medium cannot be used. Combined with PCIe AER Surprise Down error injection to trigger a controller reset, this enables complete NVMe disk hot-swap simulation where the guest sees the same device names throughout.Example usage:
drive_deldrv0 # remove backing store
drive_insertns0 disk.qcow2 # reconnect backingpcie_aer_inject_errorrp0 SDN # trigger controller resetThis approach does not delete the
--device nvme-nsbut instead replaces the BlockBackend's root node so that the existing NVMe Namespace changes the underlying storage.Questions about this approach:
1. NVMe AEN is not involved at all?
2. Does the guest have access to the new storage as soon asdrive_insertcompletes and the PCIe AER is just to kick the guest driver (e.g. getting it to update the size of the guest Linux block device)?Stefan
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
Hello Warner,
For testing namespace arrival and departure, Patch 1 (nvme-ns hotplug with AEN) is what you want. That properly executes device_del + device_add which triggers a Namespace Attribute Changed AEN, the guest sees the namespace disappear and reappear.
drive_insert is specifically for reconnecting a backing store after drive_del without the guest seeing a namespace topology change. This is useful for simulating disk removal and reinsertion, or replacing a disk, while preserving the guest's block device name, as on a real system. Different use cases, expected behavior.
Regarding FreeBSD and SDN, the SDN triggers PCIe AER recovery at the root port level. If FreeBSD just increments a counter for Surprise Down events without initiating a slot reset + controller restart, then the controller won't recover.
Linux handles SDN by freezing the device, resetting the link, and restarting the driver. FreeBSD would need similar handling for the drive_insert + SDN flow to work. But again, for namespace arrival/departure testing, you don't need SDN at all, you can use device_del/device_add and the AEN from Patch 1.
Quoted 87 lines · Warner Losh
Quoted 14 lines · Matthieu Rolla
Hello Stefan,
1. Correct, no AEN involved.The namespace stays attached to the controller throughout. Only the BlockBackend's root node changes, so from the NVMe protocol perspective nothing happened to the namespace topology.
2. Yes, the new storage is accessible immediately after
drive_insertat the QEMU block layer level. The SDN is needed to reset the NVMe controller which may have cached an error state from I/O failures during the period when no backing was present. Without the reset, the guest driver continues to see the controller as faulted. The SDN triggers the standard PCIe AER recovery path (freeze → slot reset → restart) which clears the error state and resumes normal I/O.Quoted 6 lines
One thing to note: since the namespace device stays alive, the guest kernel's block device number is preserved (e.g.
/dev/nvme0n1stays nvme0n1). This avoids theida_allocrenaming issue that occurs withdevice_del+device_addwhen filesystems hold references to the old namespace head.OK. I had hoped this would let me test my FreeBSD namespace arrival and departure messages correctly. I just added the code for AWS (and other VM) support for attaching / detaching drives and was hoping I could turn this into a regression test, but sounds like no.
I'll have to go look at what the Linux driver does here. Right now Surprise Down events just increment a counter on FreeBSD... Will thta work for a DPC system too? Or is that not relevant for the current state of the qemu art?
Warner
Quoted 49 lines
Thanks
.
*Matthieu*
www.min.io
[email protected]On Apr 14, 2026, at 7:57 PM, Stefan Hajnoczi <[email protected]> wrote:
On Thu, Apr 09, 2026 at 08:01:54AM +0200, mr-083 wrote:
Add a
drive_insertHMP command that reconnects a host block device file to an existing guest device whose backing store was previously removed withdrive_del.After
drive_del, the BlockBackend remains attached to the guest device but has no BlockDriverState (shown as "[not inserted]" in info block).drive_insertopens the specified file, finds the device's BlockBackend by iterating all backends and matching the attached device ID, then callsblk_insert_bs()to reconnect the backing store.This complements
drive_delfor non-removable devices (such as NVMe namespaces) where blockdev-change-medium cannot be used. Combined with PCIe AER Surprise Down error injection to trigger a controller reset, this enables complete NVMe disk hot-swap simulation where the guest sees the same device names throughout.Example usage:
drive_deldrv0 # remove backing store
drive_insertns0 disk.qcow2 # reconnect backingpcie_aer_inject_errorrp0 SDN # trigger controller resetThis approach does not delete the
--device nvme-nsbut instead replaces the BlockBackend's root node so that the existing NVMe Namespace changes the underlying storage.Questions about this approach:
1. NVMe AEN is not involved at all?
2. Does the guest have access to the new storage as soon asdrive_insertcompletes and the PCIe AER is just to kick the guest driver (e.g. getting it to update the size of the guest Linux block device)?Stefan
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
In reply to mr-083Quoted 21 lines · On Thu, Apr 09, 2026 at 08:01:54AM +0200, mr-083 wrote:
Add a
drive_insertHMP command that reconnects a host block device file to an existing guest device whose backing store was previously removed withdrive_del.After
drive_del, the BlockBackend remains attached to the guest device but has no BlockDriverState (shown as "[not inserted]" in info block).drive_insertopens the specified file, finds the device's BlockBackend by iterating all backends and matching the attached device ID, then callsblk_insert_bs()to reconnect the backing store.This complements
drive_delfor non-removable devices (such as NVMe namespaces) where blockdev-change-medium cannot be used. Combined with PCIe AER Surprise Down error injection to trigger a controller reset, this enables complete NVMe disk hot-swap simulation where the guest sees the same device names throughout.Example usage:
drive_deldrv0 # remove backing store
drive_insertns0 disk.qcow2 # reconnect backingpcie_aer_inject_errorrp0 SDN # trigger controller reset
Quoted 4 lines
Signed-off-by Matthieu Receveur <[email protected]>--- block/monitor/block-hmp-cmds.c | 59 ++++++++++++++++++++++++++++++++++ hmp-commands.hx | 18 +++++++++++
I see v3 has dropped this new command, but in case you have plans to re-introduce it..
First no new HMP-only commands please. Anything new must be implemented in QMP, and the HMP must be a shim to the QMP.
drive_insert semantics are somewhat overloaded - it is doing two jobs, first creating the block backend, and then associating the backend with a device. IMHO those tasks should be separate. "drive_add" or "blockdev-add" can already do the first task, so we shouldn't replicate that. AFAICT, it would only need a way to do the backend/frontend association, and possibly the PCI error injection should be done as a standard part of that re-association of backends, which would avoid the need for pcie_aer_inject_error which has no QMP impl currently.
Quoted 126 lines
include/block/block-hmp-cmds.h | 1 + 3 files changed, 78 insertions(+)block/monitor/block-hmp-cmds.c +59
-0@@ -38,7 +38,9 @@#include "qemu/osdep.h" #include "hw/core/boards.h" #include "system/block-backend.h" +#include "system/block-backend-global-state.h" #include "system/blockdev.h" +#include "block/block-global-state.h" #include "qapi/qapi-commands-block.h" #include "qapi/qapi-commands-block-export.h" #include "qobject/qdict.h" @@ -195,6 +197,63 @@ unlock:hmp_handle_error (mon, err); } +void hmp_drive_insert (Monitor *mon,const QDict *qdict) +{ +const char *id =qdict_get_str (qdict,"id" ); +const char *filename =qdict_get_str (qdict,"filename" ); +BlockBackend *blk =NULL ; +BlockBackend *iter; +BlockDriverState *bs; +Error *err =NULL ; + +GLOBAL_STATE_CODE (); + +/* +* After drive_del, the BlockBackend is removed from the monitor name +* registry but still attached to the device. Find it by iterating all +* BlockBackends and matching by the device ID shown in "info block". +*/ +for (iter =blk_all_next (NULL ); iter; iter =blk_all_next (iter)) { +DeviceState *dev =blk_get_attached_dev (iter); +if (dev && dev->id &&strcmp (dev->id, id) ==0 ) { + blk = iter; +break ; + } + } + +if (!blk) { +/* Fallback: try by block backend name */ + blk =blk_by_name (id); + } + +if (!blk) { +error_setg (&err,"Device '%s' not found" , id); + goto out; + } + +if (blk_bs (blk)) { +error_setg (&err,"Device '%s' already has a medium inserted" , id); + goto out; + } + + bs =bdrv_open (filename,NULL ,NULL ,BDRV_O_RDWR , &err); +if (!bs) { + goto out; + } + +if (blk_insert_bs (blk, bs, &err) <0 ) { +bdrv_unref (bs); + goto out; + } + +bdrv_unref (bs); +monitor_printf (mon,"OK\n" ); +return ; + +out: +hmp_handle_error (mon, err); +} +void hmp_commit (Monitor *mon,const QDict *qdict) {const char *device =qdict_get_str (qdict,"device" );hmp-commands.hx +18
-0@@ -207,6 +207,24 @@ SRST actions (drive options rerror, werror). ERST + { + .name = "drive_insert", + .args_type = "id:B,filename:F", + .params = "device filename", + .help = "insert a host block device into an empty drive", + .cmd = hmp_drive_insert, + }, + +SRST +``drive_insert`` *device* *filename* + Insert a host block device file into a drive that has been emptied by + ``drive_del``. This reconnects the backing store without removing the + guest device, enabling transparent disk hot-swap for non-removable devices + such as NVMe namespaces. Combined with PCIe AER Surprise Down error + injection (``pcie_aer_inject_error`` *device* ``SDN``), this enables + complete NVMe disk hot-swap simulation. +ERST + { .name = "change", .args_type = "device:B,force:-f,target:F,arg:s?,read-only-mode:s?",include/block/block-hmp-cmds.h +1
-0@@ -21,6 +21,7 @@ void hmp_drive_add(Monitor *mon, const QDict *qdict);void hmp_commit (Monitor *mon,const QDict *qdict);void hmp_drive_del (Monitor *mon,const QDict *qdict); +void hmp_drive_insert (Monitor *mon,const QDict *qdict);void hmp_drive_mirror (Monitor *mon,const QDict *qdict);void hmp_drive_backup (Monitor *mon,const QDict *qdict);Signature · 1 line
2.50.1 (Apple Git-155)
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com ~~ https://hachyderm.io/@berrange :| |: https://libvirt.org ~~ https://entangle-photo.org :| |: https://pixelfed.art/berrange ~~ https://fstop138.berrange.com :|
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
Thanks Daniel,
It makes sense Thanks.
Looking at the existing code, blockdev-insert-medium already does the backend/frontend association via blk_insert_bs(), but is restricted to removable devices. A new QMP command like blockdev-attach could reuse the same logic without the removable restriction, paired with blockdev-add for creating the block node.
Would that be a better approach ?
Thanks
.
Matthieu
www.min.io <>
[email protected] <>
Quoted 16 lines · Daniel P. Berrangé
see v3 has dropped this new command, but in case you have plans
to re-introduce it..First no new HMP-only commands please. Anything new must be implemented in QMP, and the HMP must be a shim to the QMP.
drive_insertsemantics are somewhat overloaded - it is doing two jobs, first creating the block backend, and then associating the backend with a device. IMHO those tasks should be separate. "drive_add" or "blockdev-add" can already do the first task, so we shouldn't replicate that. AFAICT, it would only need a way to do the backend/frontend association, and possibly the PCI error injection should be done as a standard part of that re-association of backends, which would avoid the need forpcie_aer_inject_errorwhich has no QMP impl currently.
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
Quoted 8 lines · Matthieu Rolla
Thanks Daniel,
It makes sense Thanks.
Looking at the existing code, blockdev-insert-medium already does the backend/frontend association via
blk_insert_bs(), but is restricted to removable devices. A new QMP command like blockdev-attach could reuse the same logic without the removable restriction, paired with blockdev-add for creating the block node.Would that be a better approach ?
Hi Matthieu,
I was wondering whether the blockdev needs to be changed at all. Since the disk image remains the same, is it sufficient to inject the PCIe SDN and then recover the NVMe PCI controller?
I don't understand the test scenario well enough, but it seems like you're testing at the PCIe level here rather than nothin NVMe- or blockdev-specific. Therefore blockdev commands may not be necessary.
If the testing can be done completely at the PCIe level then that would also allow other device types to be tested in the same way, which would be nice.
Stefan
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
Thanks Stefan.
I tested SDN alone on 6 disks across 3 nodes, controllers freeze, auto-recover via AER in ~200ms, and resume I/O. This works well for simulating transient PCIe errors and as you noted, it should be generic across device types.
However, SDN alone can't simulate a prolonged disk outage, the controller recovers immediately. The test scenarios need to cover:
- Transient disk failure: brief I/O disruption, automatic recovery (SDN alone works here)
- Prolonged disk failure: disk offline for minutes/hours, filesystem corruption, manual recovery required
- Disk replacement: failed disk swapped for a new one, storage stack heals data
For the last two scenarios, we need the disk to actually stay disconnected. This requires drive_del to disconnect the backing store, and blockdev-attach to reconnect it later, preserving the guest device name since drive_del has no existing counterpart for non-removable devices.
So the two series address different scenarios:
- Series 1 (namespace hotplug): namespace topology changes via device_del/device_add with AEN
- Series 2 (blockdev-attach): prolonged disk disconnect/reconnect while preserving the guest device name
Please let me know if you have any questions
.
Matthieu
www.min.io <>
[email protected] <>
Quoted 26 lines · Stefan Hajnoczi
Quoted 8 lines · Matthieu Rolla
Thanks Daniel,
It makes sense Thanks.
Looking at the existing code, blockdev-insert-medium already does the backend/frontend association via
blk_insert_bs(), but is restricted to removable devices. A new QMP command like blockdev-attach could reuse the same logic without the removable restriction, paired with blockdev-add for creating the block node.Would that be a better approach ?
Hi Matthieu,
I was wondering whether the blockdev needs to be changed at all. Since the disk image remains the same, is it sufficient to inject the PCIe SDN and then recover the NVMe PCI controller?I don't understand the test scenario well enough, but it seems like you're testing at the PCIe level here rather than nothin NVMe- or blockdev-specific. Therefore blockdev commands may not be necessary.
If the testing can be done completely at the PCIe level then that would also allow other device types to be tested in the same way, which would be nice.
Stefan
Re: [PATCH 2/2] block/monitor: add drive_insert HMP command
In reply to Daniel P. BerrangéQuoted 22 lines · Daniel P. Berrangé
Quoted 21 lines · On Thu, Apr 09, 2026 at 08:01:54AM +0200, mr-083 wrote:
Add a
drive_insertHMP command that reconnects a host block device file to an existing guest device whose backing store was previously removed withdrive_del.After
drive_del, the BlockBackend remains attached to the guest device but has no BlockDriverState (shown as "[not inserted]" in info block).drive_insertopens the specified file, finds the device's BlockBackend by iterating all backends and matching the attached device ID, then callsblk_insert_bs()to reconnect the backing store.This complements
drive_delfor non-removable devices (such as NVMe namespaces) where blockdev-change-medium cannot be used. Combined with PCIe AER Surprise Down error injection to trigger a controller reset, this enables complete NVMe disk hot-swap simulation where the guest sees the same device names throughout.Example usage:
drive_deldrv0 # remove backing store
drive_insertns0 disk.qcow2 # reconnect backingpcie_aer_inject_errorrp0 SDN # trigger controller reset
Quoted 7 lines
Quoted 4 lines
Signed-off-by Matthieu Receveur <[email protected]>--- block/monitor/block-hmp-cmds.c | 59 ++++++++++++++++++++++++++++++++++ hmp-commands.hx | 18 +++++++++++I see v3 has dropped this new command, but in case you have plans to re-introduce it..
I suggest splitting this into two separate patch series since two different use cases are being addressed:
1. --device nvme-ns hotplug using NVMe AEN. Allows users to attach and detach storage to the NVMe controller at runtime (without PCI hotplug).
2. PCIe Surprise Down and AER. Allows testing of PCI error recovery in guest drivers.
Stefan
PATCH v2 1/1 hw/nvme: add namespace hotplug support
In reply to mr-083Add hotplug support for nvme-ns devices on the NvmeBus. This enables namespace-level hot-swap without removing the NVMe controller, matching the behavior of physical NVMe drives hot-swapped in the same PCIe slot.
Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with NS_ATTR_CHANGED so the guest kernel rescans namespaces and adds the block device
- unplug: detach from all controllers, send AEN, remove from subsystem, then unrealize the device. The guest kernel rescans and removes the block device.
The plug handler skips controllers that haven't started yet (qs_created == false) to avoid interfering with boot-time namespace attachment in nvme_start_ctrl().
Both the controller bus and subsystem bus are configured as hotplug handlers via qbus_set_bus_hotplug_handler() since nvme-ns devices may reparent to the subsystem bus during realize.
Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
device_del ns0 # guest receives AEN, removes /dev/nvme0n1
drive_del drv0
drive_add 0 file=disk.qcow2,format=qcow2,id=drv0,if=none device_add nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
# guest receives AEN, adds /dev/nvme0n1
Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
--- hw/nvme/ctrl.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 1 + hw/nvme/subsys.c | 2 ++ 3 files changed, 88 insertions(+)
hw/nvme/ctrl.c +85 -0
@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h"
@@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return ;
@@ -9553,10 +9555,93 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Detach from all controllers and notify the guest via AEN. +* Must happen before unrealize to avoid use-after-free when the +* guest sends I/O to a freed namespace. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL ); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* Remove from subsystem namespace list. */ + subsys->namespaces[nsid] =NULL ; + +/* +* Unrealize: drain I/O, flush, cleanup structures, remove from QOM. +* nvme_ns_unrealize() handles drain/shutdown/cleanup internally. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )
hw/nvme/ns.c +1 -0
@@ -937,6 +937,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type =TYPE_NVME_BUS ; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ; }
hw/nvme/subsys.c +2 -0
@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h"
@@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp)NvmeSubsystem *subsys =NVME_SUBSYS (dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp); }
Signature · 1 line
2.53.0
Re: [PATCH v2] hw/nvme: add namespace hotplug support
Quoted 3 lines · On Thu, Apr 09, 2026 at 11:34:51PM +0200, mr-083 wrote:
Add hotplug support for nvme-ns devices on the NvmeBus. This enables namespace-level hot-swap without removing the NVMe controller, matching the behavior of physical NVMe drives hot-swapped in the same PCIe slot.
If we rely purely on NVMe's AEN then this is not equivalent to swapping physical drives in the same PCIe slot. Maybe adjust the wording to reflect that this is NVMe-level Namespace hotplug?
Quoted 24 lines
Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with
NS_ATTR_CHANGEDso the guest kernel rescans namespaces and adds the block device
- unplug: detach from all controllers, send AEN, remove from subsystem, then unrealize the device. The guest kernel rescans and removes the block device.The plug handler skips controllers that haven't started yet (
qs_created== false) to avoid interfering with boot-time namespace attachment innvme_start_ctrl().Both the controller bus and subsystem bus are configured as hotplug handlers via
qbus_set_bus_hotplug_handler()since nvme-ns devices may reparent to the subsystem bus during realize.Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
Quoted 8 lines
device_delns0 # guest receives AEN, removes/dev/nvme0n1
drive_deldrv0
drive_add0 file=disk.qcow2,format=qcow2,id=drv0,if=nonedevice_addnvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1# guest receives AEN, adds /dev/nvme0n1Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
Did you test a Windows Server guest? If not, I can try that next week in case there are any surprises.
Quoted 61 lines
Signed-off-by Matthieu Receveur <[email protected]>--- hw/nvme/ctrl.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 1 + hw/nvme/subsys.c | 2 ++ 3 files changed, 88 insertions(+)hw/nvme/ctrl.c +31
-0@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h" @@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return ;@@ -9553,10 +9555,93 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + }
hw/nvme/ctrl.c static const TypeInfo nvme_info = { · line 9596
Quoted 10 lines
hw/nvme/ctrl.c +10
-0+ } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i;
While there is qdev_unrealize -> nvme_ns_unrealize -> nvme_ns_drain -> blk_drain to quiesce I/O requests at the end of this function, I wonder whether it's safe to start removing the namespace before I/O has been drained.
Did you test hot unplug while the Namespace is under heavy I/O (e.g. fio job running inside the guest with lots of queued I/O requests)?
It might be necessary to stop I/O first before tearing down the namespace.
Quoted 14 lines
hw/nvme/ctrl.c +14
-0+ +/* +* Detach from all controllers and notify the guest via AEN. +* Must happen before unrealize to avoid use-after-free when the +* guest sends I/O to a freed namespace. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL );
hw/nvme/ctrl.c static const TypeInfo nvme_info = { · line 9620
Quoted 10 lines
hw/nvme/ctrl.c +10
-0+ +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* Remove from subsystem namespace list. */ + subsys->namespaces[nsid] =NULL ;
The dual of this operation is done in nvme_ns_realize():
subsys->namespaces[nsid] = ns;
Maybe nvme_ns_unrealize() should remove the namespace from the subsystem for consistency? I guess the lack of removal was never an issue before hot unplug, but now it would be nice to implement the lifecycle.
Quoted 62 lines
hw/nvme/ctrl.c +20
-0+ +/* +* Unrealize: drain I/O, flush, cleanup structures, remove from QOM. +* nvme_ns_unrealize() handles drain/shutdown/cleanup internally. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )hw/nvme/ns.c +1
-0@@ -937,6 +937,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type =TYPE_NVME_BUS ; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ; }hw/nvme/subsys.c +2
-0@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h" @@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp)NvmeSubsystem *subsys =NVME_SUBSYS (dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp); }Signature · 1 line
2.53.0
PATCH v3 1/1 hw/nvme: add namespace hotplug support
In reply to mr-083Add hotplug support for nvme-ns devices on the NvmeBus. This enables NVMe namespace-level hot-add and hot-remove via device_add and device_del with proper Asynchronous Event Notification (AEN), so the guest kernel can react to namespace topology changes.
Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with NS_ATTR_CHANGED so the guest kernel rescans namespaces and adds the block device
- unplug: drain in-flight I/O, detach from all controllers, send AEN, then unrealize the device. The guest kernel rescans and removes the block device.
The plug handler skips controllers that haven't started yet (qs_created == false) to avoid interfering with boot-time namespace attachment in nvme_start_ctrl().
The unplug handler drains in-flight I/O via nvme_ns_drain() before detaching the namespace from controllers, so pending requests can complete normally without touching freed state.
For symmetry with nvme_ns_realize() which sets subsys->namespaces[nsid], nvme_ns_unrealize() now clears that slot too — making the namespace lifecycle complete.
Both the controller bus and subsystem bus are configured as hotplug handlers via qbus_set_bus_hotplug_handler() since nvme-ns devices may reparent to the subsystem bus during realize.
Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
device_del ns0 # guest receives AEN, removes /dev/nvme0n1
drive_del drv0
drive_add 0 file=disk.qcow2,format=qcow2,id=drv0,if=none device_add nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
# guest receives AEN, adds /dev/nvme0n1
Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
--- hw/nvme/ctrl.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 8 +++++ hw/nvme/subsys.c | 2 ++ 3 files changed, 98 insertions(+)
hw/nvme/ctrl.c +88 -0
@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h"
@@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return ;
@@ -9553,10 +9555,96 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Drain in-flight I/O before tearing down the namespace. +* This must happen while the namespace is still attached to the +* controllers so any pending requests can complete normally. +*/ +nvme_ns_drain (ns); + +/* +* Detach from all controllers and notify the guest via AEN. +* The guest kernel will rescan namespaces and remove the block device. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL ); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* +* Unrealize: removes from subsystem (in nvme_ns_unrealize), flushes, +* cleans up structures, and removes from QOM. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )
hw/nvme/ns.c +8 -0
@@ -719,10 +719,17 @@ void nvme_ns_cleanup(NvmeNamespace *ns)static void nvme_ns_unrealize (DeviceState *dev) {NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid;nvme_ns_drain (ns);nvme_ns_shutdown (ns);nvme_ns_cleanup (ns); + +/* Symmetric with nvme_ns_realize() which sets subsys->namespaces[nsid]. */ +if (subsys && nsid && subsys->namespaces[nsid] == ns) { + subsys->namespaces[nsid] =NULL ; + } } void nvme_ns_atomic_configure_boundary(bool dn, uint16_t nabsn,
@@ -937,6 +944,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type =TYPE_NVME_BUS ; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ; }
hw/nvme/subsys.c +2 -0
@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h"
@@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp)NvmeSubsystem *subsys =NVME_SUBSYS (dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp); }
Signature · 1 line
2.53.0
Re: [PATCH v3] hw/nvme: add namespace hotplug support
Hello @Stefan Hajnoczi <[email protected]> ,
Thanks for the review! v3 sent.
Wording: Fixed in v3, no more "physical PCIe slot" claim. Now describes it as NVMe namespace-level hotplug.
I/O drain: Moved nvme_ns_drain() to the start of the unplug handler so in-flight I/O completes before detach. Tested under warp load (16 concurrent 1MiB uploads via MinIO/DirectPV) device_del returns in ~400ms with clean removal, no use-after-free.
Symmetry: moved subsys->namespaces[nsid] = NULL into nvme_ns_unrealize() so the namespace lifecycle is complete (mirrors what nvme_ns_realize() sets up).
I don't have a working Windows test setup, I'd really appreciate if you could test it next week as you offered.
Thanks again for your time
Quoted 231 lines · On Fri, Apr 10, 2026 at 4:29 PM mr-083 <[email protected]> wrote:
Add hotplug support for nvme-ns devices on the NvmeBus. This enables NVMe namespace-level hot-add and hot-remove via
device_addanddevice_delwith proper Asynchronous Event Notification (AEN), so the guest kernel can react to namespace topology changes.Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with
NS_ATTR_CHANGEDso the guest kernel rescans namespaces and adds the block device
- unplug: drain in-flight I/O, detach from all controllers, send AEN, then unrealize the device. The guest kernel rescans and removes the block device.The plug handler skips controllers that haven't started yet (
qs_created== false) to avoid interfering with boot-time namespace attachment innvme_start_ctrl().The unplug handler drains in-flight I/O via
nvme_ns_drain()before detaching the namespace from controllers, so pending requests can complete normally without touching freed state.For symmetry with
nvme_ns_realize()which sets subsys->namespaces[nsid],nvme_ns_unrealize()now clears that slot too — making the namespace lifecycle complete.Both the controller bus and subsystem bus are configured as hotplug handlers via
qbus_set_bus_hotplug_handler()since nvme-ns devices may reparent to the subsystem bus during realize.Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
device_delns0 # guest receives AEN, removes/dev/nvme0n1
drive_deldrv0
drive_add0 file=disk.qcow2,format=qcow2,id=drv0,if=nonedevice_addnvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1# guest receives AEN, adds /dev/nvme0n1Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
Signed-off-by Matthieu <[email protected]>--- hw/nvme/ctrl.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 8 +++++ hw/nvme/subsys.c | 2 ++ 3 files changed, 98 insertions(+)hw/nvme/ctrl.c +2
-0@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h" @@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return;hw/nvme/ctrl.c +50
-0@@ -9553,10 +9555,96 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Drain in-flight I/O before tearing down the namespace. +* This must happen while the namespace is still attached to the +* controllers so any pending requests can complete normally. +*/ +nvme_ns_drain (ns); + +/* +* Detach from all controllers and notify the guest via AEN. +* The guest kernel will rescan namespaces and remove the block device.
hw/nvme/ctrl.c +36
-0+*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL ); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* +* Unrealize: removes from subsystem (in nvme_ns_unrealize), flushes, +* cleans up structures, and removes from QOM. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )hw/nvme/ns.c +4
-0@@ -719,10 +719,17 @@ void nvme_ns_cleanup(NvmeNamespace *ns)static void nvme_ns_unrealize (DeviceState *dev) {NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid;nvme_ns_drain (ns);nvme_ns_shutdown (ns);nvme_ns_cleanup (ns); + +/* Symmetric with nvme_ns_realize() which sets subsys->namespaces[nsid]. */
hw/nvme/ns.c +4
-0+if (subsys && nsid && subsys->namespaces[nsid] == ns) { + subsys->namespaces[nsid] =NULL ; + } } void nvme_ns_atomic_configure_boundary(bool dn, uint16_t nabsn,@@ -937,6 +944,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type = TYPE_NVME_BUS; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ;}
hw/nvme/subsys.c +2
-0@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h" @@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp) NvmeSubsystem *subsys = NVME_SUBSYS(dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp);}
--
2.53.0
Stefan Hajnoczi
Reviewed-byRe: [PATCH v3] hw/nvme: add namespace hotplug support
Quoted 21 lines · Matthieu Rolla
Hello @Stefan Hajnoczi <[email protected]> ,
Thanks for the review! v3 sent.
Wording: Fixed in v3, no more "physical PCIe slot" claim. Now describes it as NVMe namespace-level hotplug.
I/O drain: Moved
nvme_ns_drain()to the start of the unplug handler so in-flight I/O completes before detach. Tested under warp load (16 concurrent 1MiB uploads via MinIO/DirectPV)device_delreturns in ~400ms with clean removal, no use-after-free.Symmetry: moved subsys->namespaces[nsid] = NULL into
nvme_ns_unrealize()so the namespace lifecycle is complete (mirrors whatnvme_ns_realize()sets up).
Quoted 4 lines
I don't have a working Windows test setup, I'd really appreciate if you could test it next week as you offered.
Thanks again for your time
Awesome, thanks!
I will give Windows Server a spin next week. I'm not an expert in hw/nvme/ but the patch looks good to me:
Re: [PATCH v3] hw/nvme: add namespace hotplug support
Hello,
I tested further more the device_del + device_add and the AEN works correctly, causing the guest to rescan namespaces.
However, when XFS filesystems are mounted on the namespace (via DirectPV/CSI in our case), the kernel doesn't reuse the block device number after re-add.
For example, nvme2n1 becomes nvme2n2 because the stale XFS mount holds a reference to the old nvme_ns_head, preventing ida_free() in nvme_free_ns_head() from releasing the instance number before the new namespace is allocated.
This causes XFS "duplicate UUID" errors since the same filesystem appears under a different device name while the old stale mount still exists.
This is a Linux kernel behavior (ida_alloc in nvme_alloc_ns_head), not a QEMU issue. The namespace hotplug patch itself is correct. But for practical use with mounted filesystems, I found that the drive_del + drive_insert approach (keeping the namespace device alive) avoids the rename entirely since no ida allocation/free cycle occurs.
Would it make sense to include drive_insert as a companion patch, or is that better handled separately ?
Thanks
Quoted 36 lines · Stefan Hajnoczi
Quoted 7 lines · Matthieu Rolla
Hello @Stefan Hajnoczi <[email protected]> ,
Thanks for the review! v3 sent.
Wording: Fixed in v3, no more "physical PCIe slot" claim. Now describes
it
Quoted 11 lines
as NVMe namespace-level hotplug.
I/O drain: Moved
nvme_ns_drain()to the start of the unplug handler so in-flight I/O completes before detach. Tested under warp load (16 concurrent 1MiB uploads via MinIO/DirectPV)device_delreturns in ~400ms with clean removal, no use-after-free.Symmetry: moved subsys->namespaces[nsid] = NULL into
nvme_ns_unrealize()so the namespace lifecycle is complete (mirrors whatnvme_ns_realize()sets
Quoted 7 lines
up).
I don't have a working Windows test setup, I'd really appreciate if you could test it next week as you offered.
Thanks again for your time
Awesome, thanks!
I will give Windows Server a spin next week. I'm not an expert in hw/nvme/ but the patch looks good to me:
Reviewed-by Stefan Hajnoczi <[email protected]>
Quoted 4 lines · On Apr 9 08:01, mr-083 wrote:
This series adds two features that together enable transparent NVMe disk hot-swap simulation in QEMU, matching the behavior of physical NVMe drives being pulled and reinserted in the same PCIe slot.
I don't understand this. From an NVMe perspective you can't hotplug a namespace. You can hotplug a PCIe-based NVM Subsystem.
Quoted 8 lines · Klaus Jensen
Quoted 4 lines · On Apr 9 08:01, mr-083 wrote:
This series adds two features that together enable transparent NVMe disk hot-swap simulation in QEMU, matching the behavior of physical NVMe drives being pulled and reinserted in the same PCIe slot.
I don't understand this. From an NVMe perspective you can't hotplug a namespace. You can hotplug a PCIe-based NVM Subsystem.
Hi Klaus,
It would be great if someone with more NVMe experience than myself can find a definite answer, but I think the Namespace List can change asynchronously even on a NVMe PCIe controller as long as it supports Namespace Management commands.
There are instances in the NVMe Express Base Specification 2.0b like:
- 8.3.1 Capacity Management Overview
"a Namespace Attribute Changed event is generated for hosts other than the host which issued the Capacity Management command"
- 8.11 Namespace Management
"If Namespace Attribute Notices are enabled, any controller(s) not processing the Namespace Management command that was attached to the namespace reports a Namespace Attribute Changed asynchronous event to the host."
I imagine this functionality would be useful in storage offload cards (IPUs/DPUs) that present as NVMe PCIe controllers instead of as NVMe-over-Fabrics. This makes sense when the host is not supposed to manage the storage itself. When the card's control plane configures a new volume, the NVMe Namespace List changes and the host is notified.
Linux and Windows NVMe PCI drivers support this according to the testing that Matthieu and I have done.
Thanks,
Stefan
Thanks for testing Windows, Stefan!
Great to have confirmation on both Linux and Windows.
Regarding drive_insert, I found that device_del + device_add works well when no filesystem is mounted on the namespace.
However, when XFS is mounted (e.g. via DirectPV/CSI), the Linux kernel doesn't reuse the block device number (nvme0n1 becomes nvme0n2) because the stale mount holds a reference to the old nvme_ns_head, preventing ida_free().
This causes XFS "duplicate UUID" errors on remount.
drive_insert avoids this by keeping the namespace device alive which means no ida cycle, same block device name.
Should I send it as a separate follow-up patch, or keep it in this series?
Matthieu
Quoted 39 lines · Stefan Hajnoczi
Quoted 8 lines · Klaus Jensen
Quoted 4 lines · On Apr 9 08:01, mr-083 wrote:
This series adds two features that together enable transparent NVMe disk hot-swap simulation in QEMU, matching the behavior of physical NVMe drives being pulled and reinserted in the same PCIe slot.
I don't understand this. From an NVMe perspective you can't hotplug a namespace. You can hotplug a PCIe-based NVM Subsystem.
Hi Klaus,
It would be great if someone with more NVMe experience than myself can find a definite answer, but I think the Namespace List can change asynchronously even on a NVMe PCIe controller as long as it supports Namespace Management commands.There are instances in the NVMe Express Base Specification 2.0b like:
- 8.3.1 Capacity Management Overview
"a Namespace Attribute Changed event is generated for hosts other than the host which issued the Capacity Management command"
- 8.11 Namespace Management
"If Namespace Attribute Notices are enabled, any controller(s) not processing the Namespace Management command that was attached to the namespace reports a Namespace Attribute Changed asynchronous event to the host."I imagine this functionality would be useful in storage offload cards (IPUs/DPUs) that present as NVMe PCIe controllers instead of as NVMe-over-Fabrics. This makes sense when the host is not supposed to manage the storage itself. When the card's control plane configures a new volume, the NVMe Namespace List changes and the host is notified.
Linux and Windows NVMe PCI drivers support this according to the testing that Matthieu and I have done.
Thanks,
Stefan
Quoted 7 lines · Matthieu Rolla
Regarding
drive_insert, I found thatdevice_del+device_addworks well when no filesystem is mounted on the namespace.However, when XFS is mounted (e.g. via DirectPV/CSI), the Linux kernel doesn't reuse the block device number (nvme0n1 becomes nvme0n2) because the stale mount holds a reference to the old
nvme_ns_head, preventingida_free().This causes XFS "duplicate UUID" errors on remount.
drive_insertavoids this by keeping the namespace device alive which means no ida cycle, same block device name.
Are you attempting some kind of covert way to swap out the backend without the host knowing you did that? Isn't that just going to confuse the filesystem that's actively using the previous backend when it's in-memory context no longer aligns with the on-disk format?
Quoted 3 lines · Matthieu Rolla
Regarding
drive_insert, I found thatdevice_del+device_addworks well when no filesystem is mounted on the namespace.However, when XFS is mounted (e.g. via DirectPV/CSI), the Linux kernel doesn't reuse the block device number (nvme0n1 becomes nvme0n2) because the stale mount holds a reference to the old
nvme_ns_head, preventingida_free().
Can you use the stable device names in /dev/disk/by-*/ instead of the /dev/nvmeCnN names to access the new namespace? Then it won't matter that ida_free() hasn't been called yet.
Quoted 1 line
This causes XFS "duplicate UUID" errors on remount.
(I have to admit that using stable device names doesn't solve this because the guest kernel still potentially has multiple XFS mounts for the file system.)
Quoted 1 line
drive_insertavoids this by keeping the namespace device alive which means no ida cycle, same block device name.
Are you sure this is safe? Even if PCIe AER somehow kills the old XFS mount, then there is still a race condition between drive_insert and PCIe AER injection when the guest kernel sees the new underlying storage through the old XFS mount.
Getting this wrong could cause data corruption, so it needs to be well understood. I don't really understand and would need to look at the guest kernel code path. Can you describe what happens to the guest kernel blkdev and the XFS mount in the drive_insert workflow?
Thanks,
Stefan
Hello Keith,
To clarify, we're not swapping to a different backend. It's the same disk file being disconnected and reconnected, simulating a physical drive being pulled and reinserted.
The sequence is:drive_del -> disconnect the backing (simulates drive pull)
User does whatever they need (test failure handling, etc.)
drive_insert -> reconnect the same backing file (simulates drive reinsertion) SDN -> reset controller so guest resumes I/O
The filesystem on disk is unchanged, same data, same UUID, same format. The guest's in-memory state realigns with the on-disk state after the controller reset, just like it would after a physical drive reinsertion on real hardware.
The use case is a storage integration lab where we need to simulate disk failures and recoveries without the guest block device being renamed, which is what happens with device_del + device_add due to the kernel's ida_alloc behavior.
Thank you
.
Matthieu
www.min.io <>
[email protected] <>
Quoted 31 lines · Stefan Hajnoczi
Quoted 3 lines · Matthieu Rolla
Regarding
drive_insert, I found thatdevice_del+device_addworks well when no filesystem is mounted on the namespace.However, when XFS is mounted (e.g. via DirectPV/CSI), the Linux kernel doesn't reuse the block device number (nvme0n1 becomes nvme0n2) because the stale mount holds a reference to the old
nvme_ns_head, preventingida_free().Can you use the stable device names in
/dev/disk/by-*/instead of the/dev/nvmeCnNnames to access the new namespace? Then it won't matter thatida_free()hasn't been called yet.Quoted 1 line
This causes XFS "duplicate UUID" errors on remount.
(I have to admit that using stable device names doesn't solve this because the guest kernel still potentially has multiple XFS mounts for the file system.)
Quoted 1 line
drive_insertavoids this by keeping the namespace device alive which means no ida cycle, same block device name.Are you sure this is safe? Even if PCIe AER somehow kills the old XFS mount, then there is still a race condition between
drive_insertand PCIe AER injection when the guest kernel sees the new underlying storage through the old XFS mount.Getting this wrong could cause data corruption, so it needs to be well understood. I don't really understand and would need to look at the guest kernel code path. Can you describe what happens to the guest kernel blkdev and the XFS mount in the
drive_insertworkflow?Thanks,
Stefan
Quoted 1 line · Matthieu Rolla
To clarify, we're not swapping to a different backend. It's the same disk file being disconnected and reconnected, simulating a physical drive being pulled and reinserted.
Is it necessary to drive_del to simulate PCIe Surprise Down? Can you perform just the PCIe actions without removing the drive from the NVMe device? That way the drive_insert command is not necessary.
Stefan
Hello,
Thanks everyone for the reviews.
I just sent v4 of the namespace hotplug patch (Series 1) with the I/O drain fix and nvme_ns_unrealize symmetry as discussed. As suggested by Stefan, the backend reassociation is sent as a separate series (Series 2).
Per Daniel's feedback, it is implemented as a QMP command (blockdev-attach) that pairs with the existing blockdev-add, with an HMP wrapper. This allows reconnecting a block node to a non-removable device's backend after drive_del, without the removable media restriction of blockdev-insert-medium.
Both patches tested with Linux 6.1 guest under DirectPV/MinIO AIStor storage stack. Scenarios covered:
Namespace attach/detach via device_del + device_add (Series 1)
Backend disconnect/reconnect via drive_del + blockdev-add + blockdev-attach + PCIe AER SDN (Series 2)
Same device name preserved across detach/attach cycles
Detach under heavy I/O (warp benchmark, 16 concurrent uploads)
Short disconnect (<3s): XFS mounts intact, DirectPV Ready, MinIO 12/12
Long disconnect (60s+): XFS journal shutdown, recovery via kubectl directpv repair, full 12/12 recovery (minio trigger healing on disk) Multiple disks across multiple nodes (6 disks, 3 nodes)
.
Matthieu
Quoted 10 lines · Stefan Hajnoczi
Quoted 1 line · Matthieu Rolla
To clarify, we're not swapping to a different backend. It's the same disk file being disconnected and reconnected, simulating a physical drive being pulled and reinserted.
Is it necessary to
drive_delto simulate PCIe Surprise Down? Can you perform just the PCIe actions without removing the drive from the NVMe device? That way thedrive_insertcommand is not necessary.Stefan
Quoted 7 lines · Stefan Hajnoczi
Quoted 6 lines · Klaus Jensen
Quoted 4 lines · On Apr 9 08:01, mr-083 wrote:
This series adds two features that together enable transparent NVMe disk hot-swap simulation in QEMU, matching the behavior of physical NVMe drives being pulled and reinserted in the same PCIe slot.
Quoted 8 lines
Quoted 2 lines
I don't understand this. From an NVMe perspective you can't hotplug a namespace. You can hotplug a PCIe-based NVM Subsystem.
Hi Klaus,
It would be great if someone with more NVMe experience than myself can find a definite answer, but I think the Namespace List can change asynchronously even on a NVMe PCIe controller as long as it supports Namespace Management commands.
I think there's some clash in terminology. From nvme protocol side, hotplug refers to bus events detected by the host, so something like PCIe slot capabilities defines how that works. This series is doing something behind the scenes from the host-controller interface visibility, so it's just coincidence that framework is also called "hotplug". From nvme protocol perspective, this patch looks like a qemu specific out-of-band method for namespace "attach/detach" via the QMP interface. Sounds fine to me: the nvme namespace events are not strictly tied to the spec defined in-band attachment status.
Adding Nilay who has done a lot of work on nvme hot plug.
Nilay please take look at these patches and let us know if they can work on powerpc
I'll set up a test bed and try this out with x86_64.
John A. Meneghini
Senior Principal Platform Storage Engineer
RHEL SST - Platform Storage Group
[email protected]
Quoted 37 lines · Stefan Hajnoczi
Quoted 8 lines · Klaus Jensen
Quoted 4 lines · On Apr 9 08:01, mr-083 wrote:
This series adds two features that together enable transparent NVMe disk hot-swap simulation in QEMU, matching the behavior of physical NVMe drives being pulled and reinserted in the same PCIe slot.
I don't understand this. From an NVMe perspective you can't hotplug a namespace. You can hotplug a PCIe-based NVM Subsystem.
Hi Klaus,
It would be great if someone with more NVMe experience than myself can find a definite answer, but I think the Namespace List can change asynchronously even on a NVMe PCIe controller as long as it supports Namespace Management commands.There are instances in the NVMe Express Base Specification 2.0b like:
- 8.3.1 Capacity Management Overview
"a Namespace Attribute Changed event is generated for hosts other than the host which issued the Capacity Management command"
- 8.11 Namespace Management
"If Namespace Attribute Notices are enabled, any controller(s) not processing the Namespace Management command that was attached to the namespace reports a Namespace Attribute Changed asynchronous event to the host."I imagine this functionality would be useful in storage offload cards (IPUs/DPUs) that present as NVMe PCIe controllers instead of as NVMe-over-Fabrics. This makes sense when the host is not supposed to manage the storage itself. When the card's control plane configures a new volume, the NVMe Namespace List changes and the host is notified.
Linux and Windows NVMe PCI drivers support this according to the testing that Matthieu and I have done.
Thanks,
Stefan
Hi John,
Quoted 6 lines · John Meneghini
Adding Nilay who has done a lot of work on nvme hot plug.
Nilay please take look at these patches and let us know if they can work on powerpc
I'll set up a test bed and try this out with
x86_64.
Thanks for looping me in.
I tested this patch series on pseries QEMU, and overall it works as expected. For the first patch (NVMe namespace hotplug), the functionality behaves correctly and achieves its intended goal. That said, from an NVMe specification perspective, the operation appears closer to a namespace attach/detach rather than a traditional “hotplug.” I understand that in the QEMU device model, this is framed as a hotplug event, which is likely why the terminology is used here, but it may still be somewhat confusing when viewed through the NVMe spec lens.
For the second patch (drive_insert), the implementation also works as intended on pseries. However, I have a concern regarding how the backend is handled. The flow effectively removes the backing storage using drive_del and later reattaches it using drive_insert. While the expectation is to reconnect the same backing store, there is currently no enforcement of this. As a result, it is possible—perhaps unintentionally—to reattach a different backing file. If this happens, it may lead to inconsistencies with the in-memory state maintained by the kernel (e.g., page cache or filesystem metadata), especially if the original device was already in use or mounted. This may potentially result in data corruption or undefined behavior from the guest’s perspective. It might be worth considering whether some form of validation or restriction should be added to ensure that the same backing store is reattached, or at least to make this behavior more explicit.
Overall, both patches are functional on pseries, but the above points may be worth addressing.
Thanks,
--Nilay
Thanks Nilay for testing on pseries!
On the terminology, agreed, v4 of the namespace patch uses "out-of-band namespace attach/detach" wording as Klaus suggested.
On the backend concern, the drive_insert patch has been replaced by a new series implementing a QMP blockdev-attach command (per Daniel's feedback).
The ability to attach a different backing file is intentional, it allows simulating disk replacement where a failed drive is swapped for a new one. The guest sees the same device name but with fresh storage.
This mirrors what happens on real hardware when you replace a failed disk in the same slot. The risk you describe (stale page cache / filesystem metadata) is expected and handled at the guest level, the filesystem detects the inconsistency and the storage stack (e.g. MinIO) heals the data via erasure coding
Link to v4 patch (serie 1): https://lists.nongnu.org/archive/html/qemu-devel/2026-04/msg02612.html
Link to new patch (serie 2): https://lists.nongnu.org/archive/html/qemu-devel/2026-04/msg02613.html
Thanks again for your time.
.
Matthieu
www.min.io <>
[email protected] <>
Quoted 37 lines · Nilay Shroff
Hi John,
Quoted 3 lines · John Meneghini
Adding Nilay who has done a lot of work on nvme hot plug.
Nilay please take look at these patches and let us know if they can work on powerpc I'll set up a test bed and try this out withx86_64.Thanks for looping me in.
I tested this patch series on pseries QEMU, and overall it works as expected. For the first patch (NVMe namespace hotplug), the functionality behaves correctly and achieves its intended goal. That said, from an NVMe specification perspective, the operation appears closer to a namespace attach/detach rather than a traditional “hotplug.” I understand that in the QEMU device model, this is framed as a hotplug event, which is likely why the terminology is used here, but it may still be somewhat confusing when viewed through the NVMe spec lens.
For the second patch (
drive_insert), the implementation also works as intended on pseries. However, I have a concern regarding how the backend is handled. The flow effectively removes the backing storage usingdrive_deland later reattaches it usingdrive_insert. While the expectation is to reconnect the same backing store, there is currently no enforcement of this. As a result, it is possible—perhaps unintentionally—to reattach a different backing file. If this happens, it may lead to inconsistencies with the in-memory state maintained by the kernel (e.g., page cache or filesystem metadata), especially if the original device was already in use or mounted. This may potentially result in data corruption or undefined behavior from the guest’s perspective. It might be worth considering whether some form of validation or restriction should be added to ensure that the same backing store is reattached, or at least to make this behavior more explicit.Overall, both patches are functional on pseries, but the above points may be worth addressing.
Thanks,
--Nilay
PATCH v4 1/1 hw/nvme: add namespace hotplug support
In reply to mr-083Add hotplug support for nvme-ns devices on the NvmeBus. This enables NVMe namespace-level hot-add and hot-remove via device_add and device_del with proper Asynchronous Event Notification (AEN), so the guest kernel can react to namespace topology changes.
Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with NS_ATTR_CHANGED so the guest kernel rescans namespaces and adds the block device
- unplug: drain in-flight I/O, detach from all controllers, send AEN, then unrealize the device. The guest kernel rescans and removes the block device.
The plug handler skips controllers that haven't started yet (qs_created == false) to avoid interfering with boot-time namespace attachment in nvme_start_ctrl().
The unplug handler drains in-flight I/O via nvme_ns_drain() before detaching the namespace from controllers, so pending requests can complete normally without touching freed state.
For symmetry with nvme_ns_realize() which sets subsys->namespaces[nsid], nvme_ns_unrealize() now clears that slot too making the namespace lifecycle complete.
Both the controller bus and subsystem bus are configured as hotplug handlers via qbus_set_bus_hotplug_handler() since nvme-ns devices may reparent to the subsystem bus during realize.
Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
device_del ns0 # guest receives AEN, removes /dev/nvme0n1
drive_del drv0
drive_add 0 file=disk.qcow2,format=qcow2,id=drv0,if=none device_add nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
# guest receives AEN, adds /dev/nvme0n1
Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
--- hw/nvme/ctrl.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 8 +++++ hw/nvme/subsys.c | 2 ++ 3 files changed, 98 insertions(+)
hw/nvme/ctrl.c +88 -0
@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h"
@@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return ;
@@ -9553,10 +9555,96 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Drain in-flight I/O before tearing down the namespace. +* This must happen while the namespace is still attached to the +* controllers so any pending requests can complete normally. +*/ +nvme_ns_drain (ns); + +/* +* Detach from all controllers and notify the guest via AEN. +* The guest kernel will rescan namespaces and remove the block device. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL ); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* +* Unrealize: removes from subsystem (in nvme_ns_unrealize), flushes, +* cleans up structures, and removes from QOM. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )
hw/nvme/ns.c +8 -0
@@ -719,10 +719,17 @@ void nvme_ns_cleanup(NvmeNamespace *ns)static void nvme_ns_unrealize (DeviceState *dev) {NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid;nvme_ns_drain (ns);nvme_ns_shutdown (ns);nvme_ns_cleanup (ns); + +/* Symmetric with nvme_ns_realize() which sets subsys->namespaces[nsid]. */ +if (subsys && nsid && subsys->namespaces[nsid] == ns) { + subsys->namespaces[nsid] =NULL ; + } } void nvme_ns_atomic_configure_boundary(bool dn, uint16_t nabsn,
@@ -937,6 +944,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type =TYPE_NVME_BUS ; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ; }
hw/nvme/subsys.c +2 -0
@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h"
@@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp)NvmeSubsystem *subsys =NVME_SUBSYS (dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp); }
Signature · 1 line
2.53.0
Re: [PATCH v4] hw/nvme: add namespace hotplug support
Quoted 46 lines · On Wed, Apr 15, 2026 at 07:38:52PM +0200, mr-083 wrote:
Add hotplug support for nvme-ns devices on the NvmeBus. This enables NVMe namespace-level hot-add and hot-remove via
device_addanddevice_delwith proper Asynchronous Event Notification (AEN), so the guest kernel can react to namespace topology changes.Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with
NS_ATTR_CHANGEDso the guest kernel rescans namespaces and adds the block device
- unplug: drain in-flight I/O, detach from all controllers, send AEN, then unrealize the device. The guest kernel rescans and removes the block device.The plug handler skips controllers that haven't started yet (
qs_created== false) to avoid interfering with boot-time namespace attachment innvme_start_ctrl().The unplug handler drains in-flight I/O via
nvme_ns_drain()before detaching the namespace from controllers, so pending requests can complete normally without touching freed state.For symmetry with
nvme_ns_realize()which sets subsys->namespaces[nsid],nvme_ns_unrealize()now clears that slot too making the namespace lifecycle complete.Both the controller bus and subsystem bus are configured as hotplug handlers via
qbus_set_bus_hotplug_handler()since nvme-ns devices may reparent to the subsystem bus during realize.Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
device_delns0 # guest receives AEN, removes/dev/nvme0n1
drive_deldrv0
drive_add0 file=disk.qcow2,format=qcow2,id=drv0,if=nonedevice_addnvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1# guest receives AEN, adds /dev/nvme0n1Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
Quoted 6 lines
Signed-off-by Matthieu <[email protected]>--- hw/nvme/ctrl.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 8 +++++ hw/nvme/subsys.c | 2 ++ 3 files changed, 98 insertions(+)
This is useful functionality because PCI hotplug is cumbersome as it requires preallocating PCIe root ports at guest creation time. Users may not know how many devices they will eventually hotplug, and so it's convenient to hotplug multiple namespaces onto an existing NVMe PCI controller.
Re: [PATCH v4] hw/nvme: add namespace hotplug support
In reply to mr-083Quoted 224 lines · On Apr 15 19:38, mr-083 wrote:
Add hotplug support for nvme-ns devices on the NvmeBus. This enables NVMe namespace-level hot-add and hot-remove via
device_addanddevice_delwith proper Asynchronous Event Notification (AEN), so the guest kernel can react to namespace topology changes.Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with
NS_ATTR_CHANGEDso the guest kernel rescans namespaces and adds the block device
- unplug: drain in-flight I/O, detach from all controllers, send AEN, then unrealize the device. The guest kernel rescans and removes the block device.The plug handler skips controllers that haven't started yet (
qs_created== false) to avoid interfering with boot-time namespace attachment innvme_start_ctrl().The unplug handler drains in-flight I/O via
nvme_ns_drain()before detaching the namespace from controllers, so pending requests can complete normally without touching freed state.For symmetry with
nvme_ns_realize()which sets subsys->namespaces[nsid],nvme_ns_unrealize()now clears that slot too making the namespace lifecycle complete.Both the controller bus and subsystem bus are configured as hotplug handlers via
qbus_set_bus_hotplug_handler()since nvme-ns devices may reparent to the subsystem bus during realize.Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
device_delns0 # guest receives AEN, removes/dev/nvme0n1
drive_deldrv0
drive_add0 file=disk.qcow2,format=qcow2,id=drv0,if=nonedevice_addnvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1# guest receives AEN, adds /dev/nvme0n1Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
Signed-off-by Matthieu <[email protected]>--- hw/nvme/ctrl.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 8 +++++ hw/nvme/subsys.c | 2 ++ 3 files changed, 98 insertions(+)hw/nvme/ctrl.c +88
-0@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h" @@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return ;@@ -9553,10 +9555,96 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Drain in-flight I/O before tearing down the namespace. +* This must happen while the namespace is still attached to the +* controllers so any pending requests can complete normally. +*/ +nvme_ns_drain (ns); + +/* +* Detach from all controllers and notify the guest via AEN. +* The guest kernel will rescan namespaces and remove the block device. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL ); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* +* Unrealize: removes from subsystem (in nvme_ns_unrealize), flushes, +* cleans up structures, and removes from QOM. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )hw/nvme/ns.c +8
-0@@ -719,10 +719,17 @@ void nvme_ns_cleanup(NvmeNamespace *ns)static void nvme_ns_unrealize (DeviceState *dev) {NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid;nvme_ns_drain (ns);nvme_ns_shutdown (ns);nvme_ns_cleanup (ns); + +/* Symmetric with nvme_ns_realize() which sets subsys->namespaces[nsid]. */ +if (subsys && nsid && subsys->namespaces[nsid] == ns) { + subsys->namespaces[nsid] =NULL ; + } } void nvme_ns_atomic_configure_boundary(bool dn, uint16_t nabsn,@@ -937,6 +944,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type =TYPE_NVME_BUS ; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ; }hw/nvme/subsys.c +2
-0@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h" @@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp)NvmeSubsystem *subsys =NVME_SUBSYS (dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp); }Signature · 1 line
2.53.0
As an out-of-band mechanism to hot-plug namespaces, this looks good.
I'll pick it up for 11.1. Thanks!
Re: [PATCH v4] hw/nvme: add namespace hotplug support
Hello Klaus,
Thank you for your feedback. I also created https://lists.nongnu.org/archive/html/qemu-devel/2026-04/msg02613.html if you are interested.
BR,
.
Matthieu
www.min.io <>
[email protected] <>
Quoted 233 lines · Klaus Jensen
Quoted 224 lines · On Apr 15 19:38, mr-083 wrote:
Add hotplug support for nvme-ns devices on the NvmeBus. This enables NVMe namespace-level hot-add and hot-remove via
device_addanddevice_delwith proper Asynchronous Event Notification (AEN), so the guest kernel can react to namespace topology changes.Mark nvme-ns devices as hotpluggable and register the NvmeBus as a hotplug handler with proper plug and unplug callbacks:
- plug: attach namespace to all started controllers and send an Asynchronous Event Notification (AEN) with
NS_ATTR_CHANGEDso the guest kernel rescans namespaces and adds the block device
- unplug: drain in-flight I/O, detach from all controllers, send AEN, then unrealize the device. The guest kernel rescans and removes the block device.The plug handler skips controllers that haven't started yet (
qs_created== false) to avoid interfering with boot-time namespace attachment innvme_start_ctrl().The unplug handler drains in-flight I/O via
nvme_ns_drain()before detaching the namespace from controllers, so pending requests can complete normally without touching freed state.For symmetry with
nvme_ns_realize()which sets subsys->namespaces[nsid],nvme_ns_unrealize()now clears that slot too making the namespace lifecycle complete.Both the controller bus and subsystem bus are configured as hotplug handlers via
qbus_set_bus_hotplug_handler()since nvme-ns devices may reparent to the subsystem bus during realize.Example hot-swap sequence using the NVMe subsystem model:
# Boot with: -device nvme-subsys,id=subsys0
# -device nvme,id=ctrl0,subsys=subsys0
# -device nvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1
device_delns0 # guest receives AEN, removes/dev/nvme0n1
drive_deldrv0
drive_add0 file=disk.qcow2,format=qcow2,id=drv0,if=nonedevice_addnvme-ns,id=ns0,drive=drv0,bus=ctrl0,nsid=1# guest receives AEN, adds /dev/nvme0n1Tested with Linux 6.1 guest (NVMe driver processes AEN and rescans namespace list automatically).
Signed-off-by Matthieu <[email protected]>---
hw/nvme/ctrl.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/nvme/ns.c | 8 +++++ hw/nvme/subsys.c | 2 ++ 3 files changed, 98 insertions(+)hw/nvme/ctrl.c +88
-0@@ -206,6 +206,7 @@#include "system/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "hw/core/qdev.h" #include "system/spdm-socket.h" #include "migration/vmstate.h" @@ -9293,6 +9294,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) }qbus_init (&n->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&n->bus));if (nvme_init_subsys (n, errp)) {return ;@@ -9553,10 +9555,96 @@ static const TypeInfo nvme_info = { }, }; +static void nvme_ns_hot_plug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Attach to all started controllers and notify via AEN. +* Skip controllers that haven't started yet (boot-time realize) — +* nvme_start_ctrl() will attach namespaces during controller init. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !ctrl->qs_created) { +continue ; + } + +if (nvme_csi_supported (ctrl, ns->csi) && !ns->params.detached) { +nvme_attach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl, ns); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + } +} + +static void nvme_ns_hot_unplug (HotplugHandler *hotplug_dev,DeviceState *dev, +Error **errp) +{ +NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid; +int i; + +/* +* Drain in-flight I/O before tearing down the namespace. +* This must happen while the namespace is still attached to the +* controllers so any pending requests can complete normally. +*/ +nvme_ns_drain (ns); + +/* +* Detach from all controllers and notify the guest via AEN. +* The guest kernel will rescan namespaces and remove the block device. +*/ +for (i =0 ; i <NVME_MAX_CONTROLLERS ; i++) { +NvmeCtrl *ctrl =nvme_subsys_ctrl (subsys, i); +if (!ctrl || !nvme_ns (ctrl, nsid)) { +continue ; + } + +nvme_detach_ns (ctrl, ns); +nvme_update_dsm_limits (ctrl,NULL ); + +if (!test_and_set_bit (nsid, ctrl->changed_nsids)) { +nvme_enqueue_event (ctrl,NVME_AER_TYPE_NOTICE , +NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED , +NVME_LOG_CHANGED_NSLIST ); + } + } + +/* +* Unrealize: removes from subsystem (in nvme_ns_unrealize), flushes, +* cleans up structures, and removes from QOM. +*/ +qdev_unrealize (dev); +} + +static void nvme_bus_class_init (ObjectClass *klass,const void *data) +{ +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (klass); + hc->plug = nvme_ns_hot_plug; + hc->unplug = nvme_ns_hot_unplug; +} +static const TypeInfo nvme_bus_info = { .name =TYPE_NVME_BUS , .parent =TYPE_BUS , .instance_size =sizeof (NvmeBus), + .class_init = nvme_bus_class_init, + .interfaces = (const InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + }, };static void nvme_register_types (void )hw/nvme/ns.c +8
-0@@ -719,10 +719,17 @@ void nvme_ns_cleanup(NvmeNamespace *ns)static void nvme_ns_unrealize (DeviceState *dev) {NvmeNamespace *ns =NVME_NS (dev); +NvmeSubsystem *subsys = ns->subsys; +uint32_t nsid = ns->params.nsid;nvme_ns_drain (ns);nvme_ns_shutdown (ns);nvme_ns_cleanup (ns); + +/* Symmetric with nvme_ns_realize() which sets subsys->namespaces[nsid]. */ +if (subsys && nsid && subsys->namespaces[nsid] == ns) { + subsys->namespaces[nsid] =NULL ; + } } void nvme_ns_atomic_configure_boundary(bool dn, uint16_t nabsn,@@ -937,6 +944,7 @@ static void nvme_ns_class_init(ObjectClass *oc, const void *data) dc->bus_type =TYPE_NVME_BUS ; dc->realize = nvme_ns_realize; dc->unrealize = nvme_ns_unrealize; + dc->hotpluggable = true;device_class_set_props (dc, nvme_ns_props); dc->desc ="Virtual NVMe namespace" ; }hw/nvme/subsys.c +2
-0@@ -9,6 +9,7 @@#include "qemu/osdep.h" #include "qemu/units.h" #include "qapi/error.h" +#include "hw/core/qdev.h" #include "nvme.h" @@ -205,6 +206,7 @@ static void nvme_subsys_realize(DeviceState *dev, Error **errp)NvmeSubsystem *subsys =NVME_SUBSYS (dev);qbus_init (&subsys->bus,sizeof (NvmeBus),TYPE_NVME_BUS , dev, dev->id); +qbus_set_bus_hotplug_handler (BUS (&subsys->bus));nvme_subsys_setup (subsys, errp); }Signature · 1 line
2.53.0As an out-of-band mechanism to hot-plug namespaces, this looks good.
Reviewed-by Klaus Jensen <[email protected]>I'll pick it up for 11.1. Thanks!