Skip to contentSkip to conversation

hw/block/nvme: support multiple namespaces

PATCH Superseded Klaus Jensen 1 participant 5 messages Jun 29, 2020

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover hw/block/nvme: support multiple namespaces

From: Klaus Jensen <[email protected]>

This adds a new 'nvme-ns' device that attaches to the nvme device through a bus. This decouples the nvme controller and nvme namespaces such that multiple namespaces may be attached to the controller.

With this in place, we can allocate a fresh pci vendor/device id and get rid of the Intel id pair that causes the Linux kernel to apply a bunch of quirks that the device no longer has.

Based-on: <[email protected]> ("[PATCH 0/3] hw/block/nvme: support scatter gather lists")

Klaus Jensen (4):
hw/block/nvme: refactor identify active namespace id list hw/block/nvme: support multiple namespaces
pci: allocate pci id for nvme
hw/block/nvme: change controller pci id

 MAINTAINERS            |   1 +
 docs/specs/nvme.txt    |  23 ++++
 docs/specs/pci-ids.txt |   1 +
 hw/block/Makefile.objs |   2 +-
 hw/block/nvme-ns.c     | 172 ++++++++++++++++++++++++++
 hw/block/nvme-ns.h     |  66 ++++++++++
 hw/block/nvme.c        | 273 +++++++++++++++++++++++++++--------------
 hw/block/nvme.h        |  45 +++----
 hw/block/trace-events  |   8 +-
 hw/core/machine.c      |   1 +
 include/hw/pci/pci.h   |   1 +
 11 files changed, 472 insertions(+), 121 deletions(-)
 create mode 100644 docs/specs/nvme.txt
 create mode 100644 hw/block/nvme-ns.c
 create mode 100644 hw/block/nvme-ns.h
Signature · 1 line
2.27.0
#1 lore raw

PATCH 1/4 hw/block/nvme: refactor identify active namespace id list

From: Klaus Jensen <[email protected]>

Prepare to support inactive namespaces.

---
 hw/block/nvme.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
hw/block/nvme.c +4 -4
index 4bcd114f76b1..eaee420219fd 100644
@@ -1573,16 +1573,16 @@ static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req)
     uint32_t min_nsid = le32_to_cpu(c->nsid);
     uint32_t *list;
     uint16_t ret;
-    int i, j = 0;
+    int j = 0;
 
     trace_pci_nvme_identify_nslist(min_nsid);
 
     list = g_malloc0(data_len);
-    for (i = 0; i < n->num_namespaces; i++) {
-        if (i < min_nsid) {
+    for (int i = 1; i <= n->num_namespaces; i++) {
+        if (i <= min_nsid) {
             continue;
         }
-        list[j++] = cpu_to_le32(i + 1);
+        list[j++] = cpu_to_le32(i);
         if (j == data_len / sizeof(uint32_t)) {
             break;
         }
Signature · 1 line
2.27.0
#2 lore raw

PATCH 2/4 hw/block/nvme: support multiple namespaces

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

This adds support for multiple namespaces by introducing a new 'nvme-ns' device model. The nvme device creates a bus named from the device name ('id'). The nvme-ns devices then connect to this and registers themselves with the nvme device.

This changes how an nvme device is created. Example with two namespaces:

-drive file=nvme0n1.img,if=none,id=disk1
-drive file=nvme0n2.img,if=none,id=disk2
-device nvme,serial=deadbeef,id=nvme0
-device nvme-ns,drive=disk1,bus=nvme0,nsid=1
-device nvme-ns,drive=disk2,bus=nvme0,nsid=2

The drive property is kept on the nvme device to keep the change backward compatible, but the property is now optional. Specifying a drive for the nvme device will always create the namespace with nsid 1.

---
 hw/block/Makefile.objs |   2 +-
 hw/block/nvme-ns.c     | 172 +++++++++++++++++++++++++++
 hw/block/nvme-ns.h     |  66 +++++++++++
 hw/block/nvme.c        | 255 ++++++++++++++++++++++++++---------------
 hw/block/nvme.h        |  44 +++----
 hw/block/trace-events  |   8 +-
 6 files changed, 431 insertions(+), 116 deletions(-)
 create mode 100644 hw/block/nvme-ns.c
 create mode 100644 hw/block/nvme-ns.h
hw/block/Makefile.objs +1 -1
index 8855c2265639..8c159bc56630 100644
@@ -13,6 +13,6 @@ common-obj-$(CONFIG_SH4) += tc58128.o
 
 obj-$(CONFIG_VIRTIO_BLK) += virtio-blk.o
 obj-$(CONFIG_VHOST_USER_BLK) += vhost-user-blk.o
-common-obj-$(CONFIG_NVME_PCI) += nvme.o
+common-obj-$(CONFIG_NVME_PCI) += nvme.o nvme-ns.o
 
 obj-y += dataplane/
hw/block/nvme-ns.c new file +172 -0

172 lines not on this page · raw message

hw/block/nvme-ns.h new file +66 -0

66 lines not on this page · raw message

hw/block/nvme.c +165 -90

544 lines not on this page · raw message

hw/block/nvme.h +23 -21

98 lines not on this page · raw message

hw/block/trace-events +4 -4
index a77f5e049bef..cbcfbfdfbafc 100644
@@ -29,6 +29,7 @@ hd_geometry_guess(void *blk, uint32_t cyls, uint32_t heads, uint32_t secs, int t
 
 # nvme.c
 # nvme traces for successful events
+pci_nvme_register_namespace(uint32_t nsid) "nsid %"PRIu32""
 pci_nvme_irq_msix(uint32_t vector) "raising MSI-X IRQ vector %u"
 pci_nvme_irq_pin(void) "pulsing IRQ pin"
 pci_nvme_irq_masked(void) "IRQ is masked"
@@ -41,9 +42,9 @@ pci_nvme_req_add_aio(uint16_t cid, void *aio, const char *blkname, uint64_t offs
 pci_nvme_aio_cb(uint16_t cid, void *aio, const char *blkname, uint64_t offset, const char *opc, void *req) "cid %"PRIu16" aio %p blk \"%s\" offset %"PRIu64" opc \"%s\" req %p"
 pci_nvme_io_cmd(uint16_t cid, uint32_t nsid, uint16_t sqid, uint8_t opcode) "cid %"PRIu16" nsid %"PRIu32" sqid %"PRIu16" opc 0x%"PRIx8""
 pci_nvme_admin_cmd(uint16_t cid, uint16_t sqid, uint8_t opcode) "cid %"PRIu16" sqid %"PRIu16" opc 0x%"PRIx8""
-pci_nvme_rw(const char *verb, uint32_t blk_count, uint64_t byte_count, uint64_t lba) "%s %"PRIu32" blocks (%"PRIu64" bytes) from LBA %"PRIu64""
-pci_nvme_rw_cb(uint16_t cid) "cid %"PRIu16""
-pci_nvme_write_zeroes(uint16_t cid, uint64_t slba, uint32_t nlb) "cid %"PRIu16" slba %"PRIu64" nlb %"PRIu32""
+pci_nvme_rw(uint16_t cid, const char *verb, uint32_t nsid, uint32_t nlb, uint64_t count, uint64_t lba) "cid %"PRIu16" %s nsid %"PRIu32" nlb %"PRIu32" count %"PRIu64" lba 0x%"PRIx64""
+pci_nvme_rw_cb(uint16_t cid, uint32_t nsid) "cid %"PRIu16" nsid %"PRIu32""
+pci_nvme_write_zeroes(uint16_t cid, uint32_t nsid, uint64_t slba, uint32_t nlb) "cid %"PRIu16" nsid %"PRIu32" slba %"PRIu64" nlb %"PRIu32""
 pci_nvme_create_sq(uint64_t addr, uint16_t sqid, uint16_t cqid, uint16_t qsize, uint16_t qflags) "create submission queue, addr=0x%"PRIx64", sqid=%"PRIu16", cqid=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16""
 pci_nvme_create_cq(uint64_t addr, uint16_t cqid, uint16_t vector, uint16_t size, uint16_t qflags, int ien) "create completion queue, addr=0x%"PRIx64", cqid=%"PRIu16", vector=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16", ien=%d"
 pci_nvme_del_sq(uint16_t qid) "deleting submission queue sqid=%"PRIu16""
@@ -100,7 +101,6 @@ pci_nvme_err_invalid_prplist_ent(uint64_t prplist) "PRP list entry is null or no
 pci_nvme_err_invalid_prp2_align(uint64_t prp2) "PRP2 is not page aligned: 0x%"PRIx64""
 pci_nvme_err_invalid_prp2_missing(void) "PRP2 is null and more data to be transferred"
 pci_nvme_err_invalid_prp(void) "invalid PRP"
-pci_nvme_err_invalid_ns(uint32_t ns, uint32_t limit) "invalid namespace %u not within 1-%u"
 pci_nvme_err_invalid_opc(uint8_t opc) "invalid opcode 0x%"PRIx8""
 pci_nvme_err_invalid_admin_opc(uint8_t opc) "invalid admin opcode 0x%"PRIx8""
 pci_nvme_err_invalid_lba_range(uint64_t start, uint64_t len, uint64_t limit) "Invalid LBA start=%"PRIu64" len=%"PRIu64" limit=%"PRIu64""
Signature · 1 line
2.27.0
#3 lore raw

PATCH 3/4 pci: allocate pci id for nvme

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

The emulated nvme device (hw/block/nvme.c) is currently using an internal Intel device id.

Prepare to change that by allocating a device id under the 1b36 (Red Hat, Inc.) vendor id.

---
 MAINTAINERS            |  1 +
 docs/specs/nvme.txt    | 23 +++++++++++++++++++++++
 docs/specs/pci-ids.txt |  1 +
 include/hw/pci/pci.h   |  1 +
 4 files changed, 26 insertions(+)
 create mode 100644 docs/specs/nvme.txt
MAINTAINERS +1 -0
index 955cc8dd5cd0..790faab64188 100644
@@ -1774,6 +1774,7 @@ L: [email protected]
 S: Supported
 F: hw/block/nvme*
 F: tests/qtest/nvme-test.c
+F: docs/specs/nvme.txt
 
 megasas
 M: Hannes Reinecke <[email protected]>
docs/specs/nvme.txt new file +23 -0
new file mode 100644
index 000000000000..56d393884e7a
@@ -0,0 +1,23 @@
+NVM Express Controller
+======================
+
+The nvme device (-device nvme) emulates an NVM Express Controller.
+
+
+Reference Specifications
+------------------------
+
+The device currently implements most mandatory features of NVMe v1.3d, see
+
+  https://nvmexpress.org/resources/specifications/
+
+for the specification.
+
+
+Known issues
+------------
+
+* The accounting numbers in the SMART/Health are reset across power cycles
+
+* Interrupt Coalescing is not supported and is disabled by default in volation
+  of the specification.
docs/specs/pci-ids.txt +1 -0
index 4d53e5c7d9d5..abbdbca6be38 100644
@@ -63,6 +63,7 @@ PCI devices (other than virtio):
 1b36:000b  PCIe Expander Bridge (-device pxb-pcie)
 1b36:000d  PCI xhci usb host adapter
 1b36:000f  mdpy (mdev sample device), linux/samples/vfio-mdev/mdpy.c
+1b36:0010  PCIe NVMe device (-device nvme)
 
 All these devices are documented in docs/specs.
 
include/hw/pci/pci.h +1 -0
index 2347dc36bfb5..7e565ba03262 100644
@@ -104,6 +104,7 @@ extern bool pci_available;
 #define PCI_DEVICE_ID_REDHAT_XHCI        0x000d
 #define PCI_DEVICE_ID_REDHAT_PCIE_BRIDGE 0x000e
 #define PCI_DEVICE_ID_REDHAT_MDPY        0x000f
+#define PCI_DEVICE_ID_REDHAT_NVME        0x0010
 #define PCI_DEVICE_ID_REDHAT_QXL         0x0100
 
 #define FMT_PCIBUS                      PRIx64
Signature · 1 line
2.27.0
#4 lore raw

PATCH 4/4 hw/block/nvme: change controller pci id

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

There are two reasons for changing this:

1. The nvme device currently uses an internal Intel device id.

2. Since commits "nvme: fix write zeroes offset and count" and "nvme:
support multiple namespaces" the controller device no longer has the quirks that the Linux kernel think it has.

As the quirks are applied based on pci vendor and device id, change them to get rid of the quirks.

To keep backward compatibility, add a new 'x-use-intel-id' parameter to the nvme device to force use of the Intel vendor and device id. This is off by default but add a compat property to set this for machines 5.0 and older.

---
 hw/block/nvme.c   | 12 ++++++++++--
 hw/block/nvme.h   |  1 +
 hw/core/machine.c |  1 +
 3 files changed, 12 insertions(+), 2 deletions(-)
hw/block/nvme.c +10 -2
index 9db8e4811433..164c5e294d06 100644
@@ -2756,6 +2756,15 @@ static void nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
 
     pci_conf[PCI_INTERRUPT_PIN] = 1;
     pci_config_set_prog_interface(pci_conf, 0x2);
+
+    if (n->params.use_intel_id) {
+        pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
+        pci_config_set_device_id(pci_conf, 0x5846);
+    } else {
+        pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
+        pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
+    }
+
     pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
     pcie_endpoint_cap_init(pci_dev, 0x80);
 
@@ -2908,6 +2917,7 @@ static Property nvme_props[] = {
     DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
     DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
     DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
+    DEFINE_PROP_BOOL("x-use-intel-id", NvmeCtrl, params.use_intel_id, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2924,8 +2934,6 @@ static void nvme_class_init(ObjectClass *oc, void *data)
     pc->realize = nvme_realize;
     pc->exit = nvme_exit;
     pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
-    pc->vendor_id = PCI_VENDOR_ID_INTEL;
-    pc->device_id = 0x5845;
     pc->revision = 2;
 
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
hw/block/nvme.h +1 -0
index 8d7610477e7d..8bf1a050497e 100644
@@ -15,6 +15,7 @@ typedef struct NvmeParams {
     uint8_t  aerl;
     uint32_t aer_max_queued;
     uint8_t  mdts;
+    bool     use_intel_id;
 } NvmeParams;
 
 typedef struct NvmeAsyncEvent {
hw/core/machine.c +1 -0
index 1d80ab0e1daf..aedf5fd48631 100644
@@ -30,6 +30,7 @@
 
 GlobalProperty hw_compat_5_0[] = {
     { "virtio-balloon-device", "page-poison", "false" },
+    { "nvme", "x-use-intel-id", "on"},
 };
 const size_t hw_compat_5_0_len = G_N_ELEMENTS(hw_compat_5_0);
 
Signature · 1 line
2.27.0