Skip to contentSkip to conversation

hw/block/nvme: support multi-path for ctrl/ns

PATCH Applied Minwoo Im 3 participants 20 messages Jan 24, 2021 – Feb 4, 2021 Arm/Amlogic Meson SoC support

lore
Share & tools
lore
author
#3 lore raw

PATCH v6 2/6 hw/block/nvme: support to map controller to a subsystem

In reply to Minwoo Im

nvme controller(nvme) can be mapped to a NVMe subsystem(nvme-subsys). This patch maps a controller to a subsystem by adding a parameter 'subsys' to the nvme device.

To map a controller to a subsystem, we need to put nvme-subsys first and then maps the subsystem to the controller:

-device nvme-subsys,id=subsys0
-device nvme,serial=foo,id=nvme0,subsys=subsys0

If 'subsys' property is not given to the nvme controller, then subsystem NQN will be created with serial (e.g., 'foo' in above example), Otherwise, it will be based on subsys id (e.g., 'subsys0' in above example).

---
 hw/block/nvme.c | 30 +++++++++++++++++++++++++-----
 hw/block/nvme.h |  3 +++
 2 files changed, 28 insertions(+), 5 deletions(-)
hw/block/nvme.c +25 -5
index aabccdf36f4b..b525fca14103 100644
@@ -22,7 +22,8 @@
  *              [pmrdev=<mem_backend_file_id>,] \
  *              max_ioqpairs=<N[optional]>, \
  *              aerl=<N[optional]>, aer_max_queued=<N[optional]>, \
- *              mdts=<N[optional]>,zoned.append_size_limit=<N[optional]> \
+ *              mdts=<N[optional]>,zoned.append_size_limit=<N[optional]>, \
+ *              subsys=<subsys_id> \
  *      -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
  *              zoned=<true|false[optional]>
  *      -device nvme-subsys,id=<subsys_id>
@@ -44,6 +45,13 @@
  *
  * nvme device parameters
  * ~~~~~~~~~~~~~~~~~~~~~~
+ * - `subsys`
+ *   NVM Subsystem device. If given, a subsystem NQN will be initialized with
+ *   <subsys_id> given. Otherwise, <serial> will be taken for subsystem NQN.
+ *   Also, it will enable multi controller capability represented in Identify
+ *   Controller data structure in CMIC (Controller Multi-path I/O and Namesapce
+ *   Sharing Capabilities), if given.
+ *
  * - `aerl`
  *   The Asynchronous Event Request Limit (AERL). Indicates the maximum number
  *   of concurrently outstanding Asynchronous Event Request commands support
@@ -4404,11 +4412,23 @@ static int nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
     return 0;
 }
 
+static void nvme_init_subnqn(NvmeCtrl *n)
+{
+    NvmeSubsystem *subsys = n->subsys;
+    NvmeIdCtrl *id = &n->id_ctrl;
+
+    if (!subsys) {
+        snprintf((char *)id->subnqn, sizeof(id->subnqn),
+                 "nqn.2019-08.org.qemu:%s", n->params.serial);
+    } else {
+        pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn);
+    }
+}
+
 static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
 {
     NvmeIdCtrl *id = &n->id_ctrl;
     uint8_t *pci_conf = pci_dev->config;
-    char *subnqn;
 
     id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
     id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
@@ -4455,9 +4475,7 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN |
                            NVME_CTRL_SGLS_BITBUCKET);
 
-    subnqn = g_strdup_printf("nqn.2019-08.org.qemu:%s", n->params.serial);
-    strpadcpy((char *)id->subnqn, sizeof(id->subnqn), subnqn, '\0');
-    g_free(subnqn);
+    nvme_init_subnqn(n);
 
     id->psd[0].mp = cpu_to_le16(0x9c4);
     id->psd[0].enlat = cpu_to_le32(0x10);
@@ -4545,6 +4563,8 @@ static Property nvme_props[] = {
     DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
     DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
                      HostMemoryBackend *),
+    DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
+                     NvmeSubsystem *),
     DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
     DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
     DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
hw/block/nvme.h +3 -0
index dee6092bd45f..04d4684601fd 100644
@@ -2,6 +2,7 @@
 #define HW_NVME_H
 
 #include "block/nvme.h"
+#include "nvme-subsys.h"
 #include "nvme-ns.h"
 
 #define NVME_MAX_NAMESPACES 256
@@ -170,6 +171,8 @@ typedef struct NvmeCtrl {
 
     uint8_t     zasl;
 
+    NvmeSubsystem   *subsys;
+
     NvmeNamespace   namespace;
     NvmeNamespace   *namespaces[NVME_MAX_NAMESPACES];
     NvmeSQueue      **sq;
Signature · 1 line
2.17.1

Minwoo Im

author
#0 lore raw

v6 cover hw/block/nvme: support multi-path for ctrl/ns

Hello,

This is sixth patch series for the support of NVMe subsystem scheme with multi-controller and namespace sharing in a subsystem.

This version has a fix in nvme_init_ctrl() when 'cntlid' is set to the Identify Controller data structure by making it by cpu_to_le16() as Keith reviewed.

Here's test result with a simple 'nvme list -v' command from this model:

  -device nvme-subsys,id=subsys0 \
  -device nvme,serial=foo,id=nvme0,subsys=subsys0 \
  -device nvme,serial=bar,id=nvme1,subsys=subsys0 \
  -device nvme,serial=baz,id=nvme2,subsys=subsys0 \
  -device nvme-ns,id=ns1,drive=drv10,nsid=1,subsys=subsys0 \
  -device nvme-ns,id=ns2,drive=drv11,nsid=2,bus=nvme2 \
  \
  -device nvme,serial=qux,id=nvme3 \
  -device nvme-ns,id=ns3,drive=drv12,nsid=3,bus=nvme3 \
  \
  -device nvme-subsys,id=subsys1 \
  -device nvme,serial=quux,id=nvme4,subsys=subsys1 \
  -device nvme-ns,id=ns4,drive=drv13,nsid=1,subsys=subsys1,zoned=true \
  root@vm:~/work# nvme list -v
  NVM Express Subsystems

Subsystem Subsystem-NQN Controllers
---------------- ------------------------------------------------------------------------------------------------ ----------------

  nvme-subsys1     nqn.2019-08.org.qemu:subsys0                                                                     nvme0, nvme1, nvme2
  nvme-subsys3     nqn.2019-08.org.qemu:qux                                                                         nvme3
  nvme-subsys4     nqn.2019-08.org.qemu:subsys1                                                                     nvme4

NVM Express Controllers

Device SN MN FR TxPort Address Subsystem Namespaces
-------- -------------------- ---------------------------------------- -------- ------ -------------- ------------ ----------------

  nvme0    foo                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:06.0   nvme-subsys1 nvme1c0n1
  nvme1    bar                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:07.0   nvme-subsys1 nvme1c1n1
  nvme2    baz                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:08.0   nvme-subsys1 nvme1c2n1, nvme1c2n2
  nvme3    qux                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:09.0   nvme-subsys3 nvme3n1
  nvme4    quux                 QEMU NVMe Ctrl                           1.0      pcie   0000:00:0a.0   nvme-subsys4 nvme4c4n1

NVM Express Namespaces

Device NSID Usage Format Controllers
------------ -------- -------------------------- ---------------- ----------------

  nvme1n1      1        134.22  MB / 134.22  MB    512   B +  0 B   nvme0, nvme1, nvme2
  nvme1n2      2        268.44  MB / 268.44  MB    512   B +  0 B   nvme2
  nvme3n1      3        268.44  MB / 268.44  MB    512   B +  0 B   nvme3
  nvme4n1      1        268.44  MB / 268.44  MB    512   B +  0 B   nvme4

Thanks,

Since V5:
- Fix endianness for 'cntlid' in Identify Controller data structure.

    (Keith)

Since V4:
- Code clean-up to snprintf rather than duplicating it and copy.

    (Keith)

- Documentation for 'subsys' clean-up. (Keith)
- Remove 'cntlid' param from nvme_init_ctrl(). (Keith)
- Put error_propagate() in nvme_realize(). (Keith)

Since RFC V3:
- Exclude 'deatched' scheme from this series. This will be covered in the next series by covering all the ns-related admin commands including ZNS and ns-mgmt. (Niklas)
- Rebased on nvme-next.
- Remove RFC tag from this V4.

Since RFC V2:
- Rebased on nvme-next branch with trivial patches from the previous version(V2) applied. (Klaus)
- Fix enumeration type name convention with NvmeIdNs prefix. (Klaus)
- Put 'cntlid' to NvmeCtrl instance in nvme_init_ctrl() which was missed in V2.
- Added 'detached' parameter to nvme-ns device to decide whether to attach or not to controller(s) in the subsystem. (Klaus)
- Implemented Identify Active Namespace ID List aprt from Identify Allocated Namespace ID List by removing fall-thru statement.

Since RFC V1:
- Updated namespace sharing scheme to be based on nvme-subsys hierarchy.

Minwoo Im (6):
hw/block/nvme: introduce nvme-subsys device hw/block/nvme: support to map controller to a subsystem hw/block/nvme: add CMIC enum value for Identify Controller hw/block/nvme: support for multi-controller in subsystem hw/block/nvme: add NMIC enum value for Identify Namespace hw/block/nvme: support for shared namespace in subsystem

 hw/block/meson.build   |   2 +-
 hw/block/nvme-ns.c     |  23 +++++++--
 hw/block/nvme-ns.h     |   7 +++
 hw/block/nvme-subsys.c | 106 +++++++++++++++++++++++++++++++++++++++++
 hw/block/nvme-subsys.h |  32 +++++++++++++
 hw/block/nvme.c        |  72 +++++++++++++++++++++++++---
 hw/block/nvme.h        |   4 ++
 include/block/nvme.h   |   8 ++++
 8 files changed, 242 insertions(+), 12 deletions(-)
 create mode 100644 hw/block/nvme-subsys.c
 create mode 100644 hw/block/nvme-subsys.h
Signature · 1 line
2.17.1
author
#1 lore raw

PATCH v6 1/6 hw/block/nvme: introduce nvme-subsys device

To support multi-path in QEMU NVMe device model, We need to have NVMe subsystem hierarchy to map controllers and namespaces to a NVMe subsystem.

This patch introduced a simple nvme-subsys device model. The subsystem will be prepared with subsystem NQN with <subsys_id> provided in nvme-subsys device:

ex) -device nvme-subsys,id=subsys0: nqn.2019-08.org.qemu:subsys0

---
 hw/block/meson.build   |  2 +-
 hw/block/nvme-subsys.c | 60 ++++++++++++++++++++++++++++++++++++++++++
 hw/block/nvme-subsys.h | 25 ++++++++++++++++++
 hw/block/nvme.c        |  3 +++
 4 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 hw/block/nvme-subsys.c
 create mode 100644 hw/block/nvme-subsys.h
hw/block/meson.build +1 -1
index 602ca6c8541d..83ea2d37978d 100644
@@ -13,7 +13,7 @@ softmmu_ss.add(when: 'CONFIG_SSI_M25P80', if_true: files('m25p80.c'))
 softmmu_ss.add(when: 'CONFIG_SWIM', if_true: files('swim.c'))
 softmmu_ss.add(when: 'CONFIG_XEN', if_true: files('xen-block.c'))
 softmmu_ss.add(when: 'CONFIG_SH4', if_true: files('tc58128.c'))
-softmmu_ss.add(when: 'CONFIG_NVME_PCI', if_true: files('nvme.c', 'nvme-ns.c'))
+softmmu_ss.add(when: 'CONFIG_NVME_PCI', if_true: files('nvme.c', 'nvme-ns.c', 'nvme-subsys.c'))
 
 specific_ss.add(when: 'CONFIG_VIRTIO_BLK', if_true: files('virtio-blk.c'))
 specific_ss.add(when: 'CONFIG_VHOST_USER_BLK', if_true: files('vhost-user-blk.c'))
hw/block/nvme-subsys.c new file +60 -0

60 lines not on this page · raw message

hw/block/nvme-subsys.h new file +25 -0
new file mode 100644
index 000000000000..40f06a4c7db0
@@ -0,0 +1,25 @@
+/*
+ * QEMU NVM Express Subsystem: nvme-subsys
+ *
+ * Copyright (c) 2021 Minwoo Im <[email protected]>
+ *
+ * This code is licensed under the GNU GPL v2.  Refer COPYING.
+ */
+
+#ifndef NVME_SUBSYS_H
+#define NVME_SUBSYS_H
+
+#define TYPE_NVME_SUBSYS "nvme-subsys"
+#define NVME_SUBSYS(obj) \
+    OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
+
+#define NVME_SUBSYS_MAX_CTRLS   32
+
+typedef struct NvmeCtrl NvmeCtrl;
+typedef struct NvmeNamespace NvmeNamespace;
+typedef struct NvmeSubsystem {
+    DeviceState parent_obj;
+    uint8_t     subnqn[256];
+} NvmeSubsystem;
+
+#endif /* NVME_SUBSYS_H */
hw/block/nvme.c +3 -0
index 21aec90637fa..aabccdf36f4b 100644
@@ -25,6 +25,7 @@
  *              mdts=<N[optional]>,zoned.append_size_limit=<N[optional]> \
  *      -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
  *              zoned=<true|false[optional]>
+ *      -device nvme-subsys,id=<subsys_id>
  *
  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
  * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
@@ -38,6 +39,8 @@
  *
  * The PMR will use BAR 4/5 exclusively.
  *
+ * To place controller(s) and namespace(s) to a subsystem, then provide
+ * nvme-subsys device as above.
  *
  * nvme device parameters
  * ~~~~~~~~~~~~~~~~~~~~~~
Signature · 1 line
2.17.1
author
#4 lore raw

PATCH v6 3/6 hw/block/nvme: add CMIC enum value for Identify Controller

In reply to Minwoo Im

Added Controller Multi-path I/O and Namespace Sharing Capabilities (CMIC) field to support multi-controller in the following patches.

This field is in Identify Controller data structure in [76].

---
 include/block/nvme.h | 4 ++++
 1 file changed, 4 insertions(+)
include/block/nvme.h +4 -0
index e4b918064df9..d6415a869c1c 100644
@@ -1034,6 +1034,10 @@ enum NvmeIdCtrlLpa {
     NVME_LPA_EXTENDED = 1 << 2,
 };
 
+enum NvmeIdCtrlCmic {
+    NVME_CMIC_MULTI_CTRL    = 1 << 1,
+};
+
 #define NVME_CTRL_SQES_MIN(sqes) ((sqes) & 0xf)
 #define NVME_CTRL_SQES_MAX(sqes) (((sqes) >> 4) & 0xf)
 #define NVME_CTRL_CQES_MIN(cqes) ((cqes) & 0xf)
Signature · 1 line
2.17.1
author
#5 lore raw

PATCH v6 4/6 hw/block/nvme: support for multi-controller in subsystem

In reply to Minwoo Im

We have nvme-subsys and nvme devices mapped together. To support multi-controller scheme to this setup, controller identifier(id) has to be managed. Earlier, cntlid(controller id) used to be always 0 because we didn't have any subsystem scheme that controller id matters.

This patch introduced 'cntlid' attribute to the nvme controller instance(NvmeCtrl) and make it allocated by the nvme-subsys device mapped to the controller. If nvme-subsys is not given to the controller, then it will always be 0 as it was.

Added 'ctrls' array in the nvme-subsys instance to manage attached controllers to the subsystem with a limit(32). This patch didn't take list for the controllers to make it seamless with nvme-ns device.

---
 hw/block/nvme-subsys.c | 21 +++++++++++++++++++++
 hw/block/nvme-subsys.h |  4 ++++
 hw/block/nvme.c        | 29 +++++++++++++++++++++++++++++
 hw/block/nvme.h        |  1 +
 4 files changed, 55 insertions(+)
hw/block/nvme-subsys.c +21 -0
index aa82911b951c..e9d61c993c90 100644
@@ -22,6 +22,27 @@
 #include "nvme.h"
 #include "nvme-subsys.h"
 
+int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp)
+{
+    NvmeSubsystem *subsys = n->subsys;
+    int cntlid;
+
+    for (cntlid = 0; cntlid < ARRAY_SIZE(subsys->ctrls); cntlid++) {
+        if (!subsys->ctrls[cntlid]) {
+            break;
+        }
+    }
+
+    if (cntlid == ARRAY_SIZE(subsys->ctrls)) {
+        error_setg(errp, "no more free controller id");
+        return -1;
+    }
+
+    subsys->ctrls[cntlid] = n;
+
+    return cntlid;
+}
+
 static void nvme_subsys_setup(NvmeSubsystem *subsys)
 {
     snprintf((char *)subsys->subnqn, sizeof(subsys->subnqn),
hw/block/nvme-subsys.h +4 -0
index 40f06a4c7db0..4eba50d96a1d 100644
@@ -20,6 +20,10 @@ typedef struct NvmeNamespace NvmeNamespace;
 typedef struct NvmeSubsystem {
     DeviceState parent_obj;
     uint8_t     subnqn[256];
+
+    NvmeCtrl    *ctrls[NVME_SUBSYS_MAX_CTRLS];
 } NvmeSubsystem;
 
+int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp);
+
 #endif /* NVME_SUBSYS_H */
hw/block/nvme.c +29 -0
index b525fca14103..7138389be4bd 100644
@@ -4435,6 +4435,9 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
     strpadcpy((char *)id->fr, sizeof(id->fr), "1.0", ' ');
     strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
+
+    id->cntlid = cpu_to_le16(n->cntlid);
+
     id->rab = 6;
     id->ieee[0] = 0x00;
     id->ieee[1] = 0x02;
@@ -4481,6 +4484,10 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->psd[0].enlat = cpu_to_le32(0x10);
     id->psd[0].exlat = cpu_to_le32(0x4);
 
+    if (n->subsys) {
+        id->cmic |= NVME_CMIC_MULTI_CTRL;
+    }
+
     NVME_CAP_SET_MQES(n->bar.cap, 0x7ff);
     NVME_CAP_SET_CQR(n->bar.cap, 1);
     NVME_CAP_SET_TO(n->bar.cap, 0xf);
@@ -4495,6 +4502,24 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     n->bar.intmc = n->bar.intms = 0;
 }
 
+static int nvme_init_subsys(NvmeCtrl *n, Error **errp)
+{
+    int cntlid;
+
+    if (!n->subsys) {
+        return 0;
+    }
+
+    cntlid = nvme_subsys_register_ctrl(n, errp);
+    if (cntlid < 0) {
+        return -1;
+    }
+
+    n->cntlid = cntlid;
+
+    return 0;
+}
+
 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 {
     NvmeCtrl *n = NVME(pci_dev);
@@ -4515,6 +4540,10 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
         return;
     }
 
+    if (nvme_init_subsys(n, errp)) {
+        error_propagate(errp, local_err);
+        return;
+    }
     nvme_init_ctrl(n, pci_dev);
 
     /* setup a namespace if the controller drive property was given */
hw/block/nvme.h +1 -0
index 04d4684601fd..b8f5f2d6ffb8 100644
@@ -134,6 +134,7 @@ typedef struct NvmeCtrl {
     NvmeBus      bus;
     BlockConf    conf;
 
+    uint16_t    cntlid;
     bool        qs_created;
     uint32_t    page_size;
     uint16_t    page_bits;
Signature · 1 line
2.17.1
author
#11 lore raw

PATCH v6 5/6 hw/block/nvme: add NMIC enum value for Identify Namespace

In reply to Minwoo Im

Added Namespace Multi-path I/O and Namespace Sharing Capabilities (NMIC) field to support shared namespace from controller(s).

This field is in Identify Namespace data structure in [30].

---
 include/block/nvme.h | 4 ++++
 1 file changed, 4 insertions(+)
include/block/nvme.h +4 -0
index d6415a869c1c..ad68cdc2b92d 100644
@@ -1203,6 +1203,10 @@ enum NvmeNsIdentifierType {
     NVME_NIDT_CSI               = 0x04,
 };
 
+enum NvmeIdNsNmic {
+    NVME_NMIC_NS_SHARED         = 1 << 0,
+};
+
 enum NvmeCsi {
     NVME_CSI_NVM                = 0x00,
     NVME_CSI_ZONED              = 0x02,
Signature · 1 line
2.17.1
author
#12 lore raw

PATCH v6 6/6 hw/block/nvme: support for shared namespace in subsystem

In reply to Minwoo Im

nvme-ns device is registered to a nvme controller device during the initialization in nvme_register_namespace() in case that 'bus' property is given which means it's mapped to a single controller.

This patch introduced a new property 'subsys' just like the controller device instance did to map a namespace to a NVMe subsystem.

If 'subsys' property is given to the nvme-ns device, it will belong to the specified subsystem and will be attached to all controllers in that subsystem by enabling shared namespace capability in NMIC(Namespace Multi-path I/O and Namespace Capabilities) in Identify Namespace.

Usage:

-device nvme-subsys,id=subsys0
-device nvme,serial=foo,id=nvme0,subsys=subsys0
-device nvme,serial=bar,id=nvme1,subsys=subsys0
-device nvme,serial=baz,id=nvme2,subsys=subsys0
-device nvme-ns,id=ns1,drive=<drv>,nsid=1,subsys=subsys0 # Shared
-device nvme-ns,id=ns2,drive=<drv>,nsid=2,bus=nvme2 # Non-shared

In the above example, 'ns1' will be shared to 'nvme0' and 'nvme1' in the same subsystem. On the other hand, 'ns2' will be attached to the 'nvme2' only as a private namespace in that subsystem.

All the namespace with 'subsys' parameter will attach all controllers in the subsystem to the namespace by default.

---
 hw/block/nvme-ns.c     | 23 ++++++++++++++++++-----
 hw/block/nvme-ns.h     |  7 +++++++
 hw/block/nvme-subsys.c | 25 +++++++++++++++++++++++++
 hw/block/nvme-subsys.h |  3 +++
 hw/block/nvme.c        | 10 +++++++++-
 5 files changed, 62 insertions(+), 6 deletions(-)
hw/block/nvme-ns.c +18 -5
index 62b25cf69bfa..9b493f2ead03 100644
@@ -63,6 +63,10 @@ static int nvme_ns_init(NvmeNamespace *ns, Error **errp)
 
     id_ns->npda = id_ns->npdg = npdg - 1;
 
+    if (nvme_ns_shared(ns)) {
+        id_ns->nmic |= NVME_NMIC_NS_SHARED;
+    }
+
     return 0;
 }
 
@@ -365,16 +369,25 @@ static void nvme_ns_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    if (nvme_register_namespace(n, ns, errp)) {
-        error_propagate_prepend(errp, local_err,
-                                "could not register namespace: ");
-        return;
+    if (ns->subsys) {
+        if (nvme_subsys_register_ns(ns, errp)) {
+            error_propagate_prepend(errp, local_err,
+                                    "could not setup namespace to subsys: ");
+            return;
+        }
+    } else {
+        if (nvme_register_namespace(n, ns, errp)) {
+            error_propagate_prepend(errp, local_err,
+                                    "could not register namespace: ");
+            return;
+        }
     }
-
 }
 
 static Property nvme_ns_props[] = {
     DEFINE_BLOCK_PROPERTIES(NvmeNamespace, blkconf),
+    DEFINE_PROP_LINK("subsys", NvmeNamespace, subsys, TYPE_NVME_SUBSYS,
+                     NvmeSubsystem *),
     DEFINE_PROP_UINT32("nsid", NvmeNamespace, params.nsid, 0),
     DEFINE_PROP_UUID("uuid", NvmeNamespace, params.uuid),
     DEFINE_PROP_BOOL("zoned", NvmeNamespace, params.zoned, false),
hw/block/nvme-ns.h +7 -0
index 293ac990e3f6..929e78861903 100644
@@ -47,6 +47,8 @@ typedef struct NvmeNamespace {
     const uint32_t *iocs;
     uint8_t      csi;
 
+    NvmeSubsystem   *subsys;
+
     NvmeIdNsZoned   *id_ns_zoned;
     NvmeZone        *zone_array;
     QTAILQ_HEAD(, NvmeZone) exp_open_zones;
@@ -77,6 +79,11 @@ static inline uint32_t nvme_nsid(NvmeNamespace *ns)
     return -1;
 }
 
+static inline bool nvme_ns_shared(NvmeNamespace *ns)
+{
+    return !!ns->subsys;
+}
+
 static inline NvmeLBAF *nvme_ns_lbaf(NvmeNamespace *ns)
 {
     NvmeIdNs *id_ns = &ns->id_ns;
hw/block/nvme-subsys.c +25 -0
index e9d61c993c90..641de33e99fc 100644
@@ -43,6 +43,31 @@ int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp)
     return cntlid;
 }
 
+int nvme_subsys_register_ns(NvmeNamespace *ns, Error **errp)
+{
+    NvmeSubsystem *subsys = ns->subsys;
+    NvmeCtrl *n;
+    int i;
+
+    if (subsys->namespaces[nvme_nsid(ns)]) {
+        error_setg(errp, "namespace %d already registerd to subsy %s",
+                   nvme_nsid(ns), subsys->parent_obj.id);
+        return -1;
+    }
+
+    subsys->namespaces[nvme_nsid(ns)] = ns;
+
+    for (i = 0; i < ARRAY_SIZE(subsys->ctrls); i++) {
+        n = subsys->ctrls[i];
+
+        if (n && nvme_register_namespace(n, ns, errp)) {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
 static void nvme_subsys_setup(NvmeSubsystem *subsys)
 {
     snprintf((char *)subsys->subnqn, sizeof(subsys->subnqn),
hw/block/nvme-subsys.h +3 -0
index 4eba50d96a1d..ccf6a71398d3 100644
@@ -14,6 +14,7 @@
     OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
 
 #define NVME_SUBSYS_MAX_CTRLS   32
+#define NVME_SUBSYS_MAX_NAMESPACES  32
 
 typedef struct NvmeCtrl NvmeCtrl;
 typedef struct NvmeNamespace NvmeNamespace;
@@ -22,8 +23,10 @@ typedef struct NvmeSubsystem {
     uint8_t     subnqn[256];
 
     NvmeCtrl    *ctrls[NVME_SUBSYS_MAX_CTRLS];
+    NvmeNamespace *namespaces[NVME_SUBSYS_MAX_NAMESPACES];
 } NvmeSubsystem;
 
 int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp);
+int nvme_subsys_register_ns(NvmeNamespace *ns, Error **errp);
 
 #endif /* NVME_SUBSYS_H */
hw/block/nvme.c +9 -1
index 7138389be4bd..8259dbf48ec0 100644
@@ -25,7 +25,8 @@
  *              mdts=<N[optional]>,zoned.append_size_limit=<N[optional]>, \
  *              subsys=<subsys_id> \
  *      -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
- *              zoned=<true|false[optional]>
+ *              zoned=<true|false[optional]>, \
+ *              subsys=<subsys_id>
  *      -device nvme-subsys,id=<subsys_id>
  *
  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
@@ -70,6 +71,13 @@
  *   data size being in effect. By setting this property to 0, users can make
  *   ZASL to be equal to MDTS. This property only affects zoned namespaces.
  *
+ * nvme namespace device parameters
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ * - `subsys`
+ *   NVM Subsystem device.  If given, this namespace will be attached to all
+ *   controllers in the subsystem. Otherwise, `bus` must be given to attach
+ *   this namespace to a specified single controller as a non-shared namespace.
+ *
  * Setting `zoned` to true selects Zoned Command Set at the namespace.
  * In this case, the following namespace properties are available to configure
  * zoned operation:
Signature · 1 line
2.17.1
#6 lore raw

Re: [PATCH V6 4/6] hw/block/nvme: support for multi-controller in subsystem

In reply to Minwoo Im
Quoted 27 lines · Minwoo Im

We have nvme-subsys and nvme devices mapped together. To support multi-controller scheme to this setup, controller identifier(id) has to be managed. Earlier, cntlid(controller id) used to be always 0 because we didn't have any subsystem scheme that controller id matters.

This patch introduced 'cntlid' attribute to the nvme controller instance(NvmeCtrl) and make it allocated by the nvme-subsys device mapped to the controller. If nvme-subsys is not given to the controller, then it will always be 0 as it was.

Added 'ctrls' array in the nvme-subsys instance to manage attached controllers to the subsystem with a limit(32). This patch didn't take list for the controllers to make it seamless with nvme-ns device.

---
 hw/block/nvme-subsys.c | 21 +++++++++++++++++++++
 hw/block/nvme-subsys.h |  4 ++++
 hw/block/nvme.c        | 29 +++++++++++++++++++++++++++++
 hw/block/nvme.h        |  1 +
 4 files changed, 55 insertions(+)
hw/block/nvme.c +0 -0
index b525fca14103..7138389be4bd 100644

hw/block/nvme.c static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) · line 4489

Quoted 6 lines
hw/block/nvme.c +3 -0
@@ -4481,6 +4484,10 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->psd[0].enlat = cpu_to_le32(0x10);
     id->psd[0].exlat = cpu_to_le32(0x4);
 
+    if (n->subsys) {
+        id->cmic |= NVME_CMIC_MULTI_CTRL;
+    }

Since multiple controllers show up with a PCIe port of their own, do we need to set bit 0 (NVME_CMIC_MULTI_PORT?) as well? Or am I misunderstanding that bit?

#7 lore raw

Re: [PATCH V6 4/6] hw/block/nvme: support for multi-controller in subsystem

Quoted 31 lines · Klaus Jensen
Quoted 30 lines · Minwoo Im

We have nvme-subsys and nvme devices mapped together. To support multi-controller scheme to this setup, controller identifier(id) has to be managed. Earlier, cntlid(controller id) used to be always 0 because we didn't have any subsystem scheme that controller id matters.

This patch introduced 'cntlid' attribute to the nvme controller instance(NvmeCtrl) and make it allocated by the nvme-subsys device mapped to the controller. If nvme-subsys is not given to the controller, then it will always be 0 as it was.

Added 'ctrls' array in the nvme-subsys instance to manage attached controllers to the subsystem with a limit(32). This patch didn't take list for the controllers to make it seamless with nvme-ns device.

---
 hw/block/nvme-subsys.c | 21 +++++++++++++++++++++
 hw/block/nvme-subsys.h |  4 ++++
 hw/block/nvme.c        | 29 +++++++++++++++++++++++++++++
 hw/block/nvme.h        |  1 +
 4 files changed, 55 insertions(+)
hw/block/nvme.c +0 -0
index b525fca14103..7138389be4bd 100644
@@ -4481,6 +4484,10 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->psd[0].enlat = cpu_to_le32(0x10);
     id->psd[0].exlat = cpu_to_le32(0x4);
 
Quoted 7 lines

hw/block/nvme.c static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) · line 4489

Quoted 3 lines
hw/block/nvme.c +3 -0
+    if (n->subsys) {
+        id->cmic |= NVME_CMIC_MULTI_CTRL;
+    }

Since multiple controllers show up with a PCIe port of their own, do we need to set bit 0 (NVME_CMIC_MULTI_PORT?) as well? Or am I misunderstanding that bit?

AIUI, if you report this MULTI_PORT bit, then each PCI device in the subsystem needs to report a different "Port Number" in their PCIe Link Capabilities register. I don't think we can manipulate that value from the nvme "device", but I also don't know what a host should do with this information even if we could. So, I think it's safe to leave it at 0.

#13 lore raw

Re: [PATCH V6 6/6] hw/block/nvme: support for shared namespace in subsystem

In reply to Minwoo Im
Quoted 114 lines · Minwoo Im

nvme-ns device is registered to a nvme controller device during the initialization in nvme_register_namespace() in case that 'bus' property is given which means it's mapped to a single controller.

This patch introduced a new property 'subsys' just like the controller device instance did to map a namespace to a NVMe subsystem.

If 'subsys' property is given to the nvme-ns device, it will belong to the specified subsystem and will be attached to all controllers in that subsystem by enabling shared namespace capability in NMIC(Namespace Multi-path I/O and Namespace Capabilities) in Identify Namespace.

Usage:

-device nvme-subsys,id=subsys0
-device nvme,serial=foo,id=nvme0,subsys=subsys0
-device nvme,serial=bar,id=nvme1,subsys=subsys0
-device nvme,serial=baz,id=nvme2,subsys=subsys0
-device nvme-ns,id=ns1,drive=<drv>,nsid=1,subsys=subsys0 # Shared
-device nvme-ns,id=ns2,drive=<drv>,nsid=2,bus=nvme2 # Non-shared

In the above example, 'ns1' will be shared to 'nvme0' and 'nvme1' in the same subsystem. On the other hand, 'ns2' will be attached to the 'nvme2' only as a private namespace in that subsystem.

All the namespace with 'subsys' parameter will attach all controllers in the subsystem to the namespace by default.

---
 hw/block/nvme-ns.c     | 23 ++++++++++++++++++-----
 hw/block/nvme-ns.h     |  7 +++++++
 hw/block/nvme-subsys.c | 25 +++++++++++++++++++++++++
 hw/block/nvme-subsys.h |  3 +++
 hw/block/nvme.c        | 10 +++++++++-
 5 files changed, 62 insertions(+), 6 deletions(-)
hw/block/nvme-ns.c +18 -5
index 62b25cf69bfa..9b493f2ead03 100644
@@ -63,6 +63,10 @@ static int nvme_ns_init(NvmeNamespace *ns, Error **errp)
 
     id_ns->npda = id_ns->npdg = npdg - 1;
 
+    if (nvme_ns_shared(ns)) {
+        id_ns->nmic |= NVME_NMIC_NS_SHARED;
+    }
+
     return 0;
 }
 
@@ -365,16 +369,25 @@ static void nvme_ns_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    if (nvme_register_namespace(n, ns, errp)) {
-        error_propagate_prepend(errp, local_err,
-                                "could not register namespace: ");
-        return;
+    if (ns->subsys) {
+        if (nvme_subsys_register_ns(ns, errp)) {
+            error_propagate_prepend(errp, local_err,
+                                    "could not setup namespace to subsys: ");
+            return;
+        }
+    } else {
+        if (nvme_register_namespace(n, ns, errp)) {
+            error_propagate_prepend(errp, local_err,
+                                    "could not register namespace: ");
+            return;
+        }
     }
-
 }
 
 static Property nvme_ns_props[] = {
     DEFINE_BLOCK_PROPERTIES(NvmeNamespace, blkconf),
+    DEFINE_PROP_LINK("subsys", NvmeNamespace, subsys, TYPE_NVME_SUBSYS,
+                     NvmeSubsystem *),
     DEFINE_PROP_UINT32("nsid", NvmeNamespace, params.nsid, 0),
     DEFINE_PROP_UUID("uuid", NvmeNamespace, params.uuid),
     DEFINE_PROP_BOOL("zoned", NvmeNamespace, params.zoned, false),
hw/block/nvme-ns.h +7 -0
index 293ac990e3f6..929e78861903 100644
@@ -47,6 +47,8 @@ typedef struct NvmeNamespace {
     const uint32_t *iocs;
     uint8_t      csi;
 
+    NvmeSubsystem   *subsys;
+
     NvmeIdNsZoned   *id_ns_zoned;
     NvmeZone        *zone_array;
     QTAILQ_HEAD(, NvmeZone) exp_open_zones;
@@ -77,6 +79,11 @@ static inline uint32_t nvme_nsid(NvmeNamespace *ns)
     return -1;
 }
 
+static inline bool nvme_ns_shared(NvmeNamespace *ns)
+{
+    return !!ns->subsys;
+}
+
 static inline NvmeLBAF *nvme_ns_lbaf(NvmeNamespace *ns)
 {
     NvmeIdNs *id_ns = &ns->id_ns;
hw/block/nvme-subsys.c +0 -0
index e9d61c993c90..641de33e99fc 100644
@@ -43,6 +43,31 @@ int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp)
     return cntlid;

hw/block/nvme-subsys.c int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp) · line 53

Quoted 10 lines
hw/block/nvme-subsys.c +8 -0
 }
 
+int nvme_subsys_register_ns(NvmeNamespace *ns, Error **errp)
+{
+    NvmeSubsystem *subsys = ns->subsys;
+    NvmeCtrl *n;
+    int i;
+
+    if (subsys->namespaces[nvme_nsid(ns)]) {
+        error_setg(errp, "namespace %d already registerd to subsy %s",

Small typo.

"registerEd to subsySTEM"

Quoted 74 lines
hw/block/nvme-subsys.c +17 -0
+                   nvme_nsid(ns), subsys->parent_obj.id);
+        return -1;
+    }
+
+    subsys->namespaces[nvme_nsid(ns)] = ns;
+
+    for (i = 0; i < ARRAY_SIZE(subsys->ctrls); i++) {
+        n = subsys->ctrls[i];
+
+        if (n && nvme_register_namespace(n, ns, errp)) {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
 static void nvme_subsys_setup(NvmeSubsystem *subsys)
 {
     snprintf((char *)subsys->subnqn, sizeof(subsys->subnqn),
hw/block/nvme-subsys.h +3 -0
index 4eba50d96a1d..ccf6a71398d3 100644
@@ -14,6 +14,7 @@
     OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
 
 #define NVME_SUBSYS_MAX_CTRLS   32
+#define NVME_SUBSYS_MAX_NAMESPACES  32
 
 typedef struct NvmeCtrl NvmeCtrl;
 typedef struct NvmeNamespace NvmeNamespace;
@@ -22,8 +23,10 @@ typedef struct NvmeSubsystem {
     uint8_t     subnqn[256];
 
     NvmeCtrl    *ctrls[NVME_SUBSYS_MAX_CTRLS];
+    NvmeNamespace *namespaces[NVME_SUBSYS_MAX_NAMESPACES];
 } NvmeSubsystem;
 
 int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp);
+int nvme_subsys_register_ns(NvmeNamespace *ns, Error **errp);
 
 #endif /* NVME_SUBSYS_H */
hw/block/nvme.c +9 -1
index 7138389be4bd..8259dbf48ec0 100644
@@ -25,7 +25,8 @@
  *              mdts=<N[optional]>,zoned.append_size_limit=<N[optional]>, \
  *              subsys=<subsys_id> \
  *      -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
- *              zoned=<true|false[optional]>
+ *              zoned=<true|false[optional]>, \
+ *              subsys=<subsys_id>
  *      -device nvme-subsys,id=<subsys_id>
  *
  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
@@ -70,6 +71,13 @@
  *   data size being in effect. By setting this property to 0, users can make
  *   ZASL to be equal to MDTS. This property only affects zoned namespaces.
  *
+ * nvme namespace device parameters
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ * - `subsys`
+ *   NVM Subsystem device.  If given, this namespace will be attached to all
+ *   controllers in the subsystem. Otherwise, `bus` must be given to attach
+ *   this namespace to a specified single controller as a non-shared namespace.
+ *
  * Setting `zoned` to true selects Zoned Command Set at the namespace.
  * In this case, the following namespace properties are available to configure
  * zoned operation:
Signature · 1 line
2.17.1
#2 lore raw

Re: [PATCH V6 1/6] hw/block/nvme: introduce nvme-subsys device

In reply to Minwoo Im
Quoted 53 lines · Minwoo Im

To support multi-path in QEMU NVMe device model, We need to have NVMe subsystem hierarchy to map controllers and namespaces to a NVMe subsystem.

This patch introduced a simple nvme-subsys device model. The subsystem will be prepared with subsystem NQN with <subsys_id> provided in nvme-subsys device:

ex) -device nvme-subsys,id=subsys0: nqn.2019-08.org.qemu:subsys0

---
 hw/block/meson.build   |  2 +-
 hw/block/nvme-subsys.c | 60 ++++++++++++++++++++++++++++++++++++++++++
 hw/block/nvme-subsys.h | 25 ++++++++++++++++++
 hw/block/nvme.c        |  3 +++
 4 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 hw/block/nvme-subsys.c
 create mode 100644 hw/block/nvme-subsys.h
hw/block/meson.build +1 -1
index 602ca6c8541d..83ea2d37978d 100644
@@ -13,7 +13,7 @@ softmmu_ss.add(when: 'CONFIG_SSI_M25P80', if_true: files('m25p80.c'))
 softmmu_ss.add(when: 'CONFIG_SWIM', if_true: files('swim.c'))
 softmmu_ss.add(when: 'CONFIG_XEN', if_true: files('xen-block.c'))
 softmmu_ss.add(when: 'CONFIG_SH4', if_true: files('tc58128.c'))
-softmmu_ss.add(when: 'CONFIG_NVME_PCI', if_true: files('nvme.c', 'nvme-ns.c'))
+softmmu_ss.add(when: 'CONFIG_NVME_PCI', if_true: files('nvme.c', 'nvme-ns.c', 'nvme-subsys.c'))
 
 specific_ss.add(when: 'CONFIG_VIRTIO_BLK', if_true: files('virtio-blk.c'))
 specific_ss.add(when: 'CONFIG_VHOST_USER_BLK', if_true: files('vhost-user-blk.c'))
hw/block/nvme-subsys.c new file +14 -0
new file mode 100644
index 000000000000..aa82911b951c
@@ -0,0 +1,60 @@
+/*
+ * QEMU NVM Express Subsystem: nvme-subsys
+ *
+ * Copyright (c) 2021 Minwoo Im <[email protected]>
+ *
+ * This code is licensed under the GNU GPL v2.  Refer COPYING.
+ */
+
+#include "qemu/units.h"
+#include "qemu/osdep.h"
+#include "qemu/uuid.h"
+#include "qemu/iov.h"
+#include "qemu/cutils.h"
+#include "qapi/error.h"

hw/block/nvme-subsys.c line 24

Quoted 10 lines
hw/block/nvme-subsys.c new file +10 -0
+#include "hw/qdev-properties.h"
+#include "hw/qdev-core.h"
+#include "hw/block/block.h"
+#include "block/aio.h"
+#include "block/accounting.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "nvme.h"
+#include "nvme-subsys.h"
+

This can be slimmed down to

#include "qemu/osdep.h"
#include "qapi/error.h"
#include "hw/block/block.h"
#include "sysemu/sysemu.h"
#include "hw/pci/pci.h"
#include "nvme.h"
#include "nvme-subsys.h"

Tested-by Reviewed-by on the v6 series
#14 lore raw
In reply to Minwoo Im
Quoted 96 lines · Minwoo Im

Hello,

This is sixth patch series for the support of NVMe subsystem scheme with multi-controller and namespace sharing in a subsystem.

This version has a fix in nvme_init_ctrl() when 'cntlid' is set to the Identify Controller data structure by making it by cpu_to_le16() as Keith reviewed.

Here's test result with a simple 'nvme list -v' command from this model:

  -device nvme-subsys,id=subsys0 \
  -device nvme,serial=foo,id=nvme0,subsys=subsys0 \
  -device nvme,serial=bar,id=nvme1,subsys=subsys0 \
  -device nvme,serial=baz,id=nvme2,subsys=subsys0 \
  -device nvme-ns,id=ns1,drive=drv10,nsid=1,subsys=subsys0 \
  -device nvme-ns,id=ns2,drive=drv11,nsid=2,bus=nvme2 \
  \
  -device nvme,serial=qux,id=nvme3 \
  -device nvme-ns,id=ns3,drive=drv12,nsid=3,bus=nvme3 \
  \
  -device nvme-subsys,id=subsys1 \
  -device nvme,serial=quux,id=nvme4,subsys=subsys1 \
  -device nvme-ns,id=ns4,drive=drv13,nsid=1,subsys=subsys1,zoned=true \
  root@vm:~/work# nvme list -v
  NVM Express Subsystems

Subsystem Subsystem-NQN Controllers
---------------- ------------------------------------------------------------------------------------------------ ----------------

  nvme-subsys1     nqn.2019-08.org.qemu:subsys0                                                                     nvme0, nvme1, nvme2
  nvme-subsys3     nqn.2019-08.org.qemu:qux                                                                         nvme3
  nvme-subsys4     nqn.2019-08.org.qemu:subsys1                                                                     nvme4

NVM Express Controllers

Device SN MN FR TxPort Address Subsystem Namespaces
-------- -------------------- ---------------------------------------- -------- ------ -------------- ------------ ----------------

  nvme0    foo                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:06.0   nvme-subsys1 nvme1c0n1
  nvme1    bar                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:07.0   nvme-subsys1 nvme1c1n1
  nvme2    baz                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:08.0   nvme-subsys1 nvme1c2n1, nvme1c2n2
  nvme3    qux                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:09.0   nvme-subsys3 nvme3n1
  nvme4    quux                 QEMU NVMe Ctrl                           1.0      pcie   0000:00:0a.0   nvme-subsys4 nvme4c4n1

NVM Express Namespaces

Device NSID Usage Format Controllers
------------ -------- -------------------------- ---------------- ----------------

  nvme1n1      1        134.22  MB / 134.22  MB    512   B +  0 B   nvme0, nvme1, nvme2
  nvme1n2      2        268.44  MB / 268.44  MB    512   B +  0 B   nvme2
  nvme3n1      3        268.44  MB / 268.44  MB    512   B +  0 B   nvme3
  nvme4n1      1        268.44  MB / 268.44  MB    512   B +  0 B   nvme4

Thanks,

Since V5:
- Fix endianness for 'cntlid' in Identify Controller data structure.

    (Keith)

Since V4:
- Code clean-up to snprintf rather than duplicating it and copy.

    (Keith)

- Documentation for 'subsys' clean-up. (Keith)
- Remove 'cntlid' param from nvme_init_ctrl(). (Keith)
- Put error_propagate() in nvme_realize(). (Keith)

Since RFC V3:
- Exclude 'deatched' scheme from this series. This will be covered in the next series by covering all the ns-related admin commands including ZNS and ns-mgmt. (Niklas)
- Rebased on nvme-next.
- Remove RFC tag from this V4.

Since RFC V2:
- Rebased on nvme-next branch with trivial patches from the previous version(V2) applied. (Klaus)
- Fix enumeration type name convention with NvmeIdNs prefix. (Klaus)
- Put 'cntlid' to NvmeCtrl instance in nvme_init_ctrl() which was missed in V2.
- Added 'detached' parameter to nvme-ns device to decide whether to attach or not to controller(s) in the subsystem. (Klaus)
- Implemented Identify Active Namespace ID List aprt from Identify Allocated Namespace ID List by removing fall-thru statement.

Since RFC V1:
- Updated namespace sharing scheme to be based on nvme-subsys hierarchy.

Minwoo Im (6):
hw/block/nvme: introduce nvme-subsys device hw/block/nvme: support to map controller to a subsystem hw/block/nvme: add CMIC enum value for Identify Controller hw/block/nvme: support for multi-controller in subsystem hw/block/nvme: add NMIC enum value for Identify Namespace hw/block/nvme: support for shared namespace in subsystem

Quoted 12 lines
 hw/block/meson.build   |   2 +-
 hw/block/nvme-ns.c     |  23 +++++++--
 hw/block/nvme-ns.h     |   7 +++
 hw/block/nvme-subsys.c | 106 +++++++++++++++++++++++++++++++++++++++++
 hw/block/nvme-subsys.h |  32 +++++++++++++
 hw/block/nvme.c        |  72 +++++++++++++++++++++++++---
 hw/block/nvme.h        |   4 ++
 include/block/nvme.h   |   8 ++++
 8 files changed, 242 insertions(+), 12 deletions(-)
 create mode 100644 hw/block/nvme-subsys.c
 create mode 100644 hw/block/nvme-subsys.h

Thanks Minwoo! This really is great stuff.

Notwithstanding the nitpicks I've pointed out for [1/6] and [6/6] (no need to v7 for those), take my

author
#8 lore raw

Re: [PATCH V6 4/6] hw/block/nvme: support for multi-controller in subsystem

In reply to Keith Busch
Quoted 40 lines · Keith Busch
Quoted 31 lines · Klaus Jensen
Quoted 30 lines · Minwoo Im

We have nvme-subsys and nvme devices mapped together. To support multi-controller scheme to this setup, controller identifier(id) has to be managed. Earlier, cntlid(controller id) used to be always 0 because we didn't have any subsystem scheme that controller id matters.

This patch introduced 'cntlid' attribute to the nvme controller instance(NvmeCtrl) and make it allocated by the nvme-subsys device mapped to the controller. If nvme-subsys is not given to the controller, then it will always be 0 as it was.

Added 'ctrls' array in the nvme-subsys instance to manage attached controllers to the subsystem with a limit(32). This patch didn't take list for the controllers to make it seamless with nvme-ns device.

---
 hw/block/nvme-subsys.c | 21 +++++++++++++++++++++
 hw/block/nvme-subsys.h |  4 ++++
 hw/block/nvme.c        | 29 +++++++++++++++++++++++++++++
 hw/block/nvme.h        |  1 +
 4 files changed, 55 insertions(+)
hw/block/nvme.c +0 -0
index b525fca14103..7138389be4bd 100644
@@ -4481,6 +4484,10 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->psd[0].enlat = cpu_to_le32(0x10);
     id->psd[0].exlat = cpu_to_le32(0x4);
 
Quoted 7 lines

hw/block/nvme.c static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) · line 4489

Quoted 3 lines
hw/block/nvme.c +3 -0
+    if (n->subsys) {
+        id->cmic |= NVME_CMIC_MULTI_CTRL;
+    }

Since multiple controllers show up with a PCIe port of their own, do we need to set bit 0 (NVME_CMIC_MULTI_PORT?) as well? Or am I misunderstanding that bit?

Quoted 5 lines

AIUI, if you report this MULTI_PORT bit, then each PCI device in the subsystem needs to report a different "Port Number" in their PCIe Link Capabilities register. I don't think we can manipulate that value from the nvme "device", but I also don't know what a host should do with this information even if we could. So, I think it's safe to leave it at 0.

AFAIK, If we leave it to 0, kernel will not allocate disk for multi-path case (e.g., nvmeXcYnZ).

author
#15 lore raw
In reply to Klaus Jensen
Quoted 110 lines · Klaus Jensen
Quoted 96 lines · Minwoo Im

Hello,

This is sixth patch series for the support of NVMe subsystem scheme with multi-controller and namespace sharing in a subsystem.

This version has a fix in nvme_init_ctrl() when 'cntlid' is set to the Identify Controller data structure by making it by cpu_to_le16() as Keith reviewed.

Here's test result with a simple 'nvme list -v' command from this model:

  -device nvme-subsys,id=subsys0 \
  -device nvme,serial=foo,id=nvme0,subsys=subsys0 \
  -device nvme,serial=bar,id=nvme1,subsys=subsys0 \
  -device nvme,serial=baz,id=nvme2,subsys=subsys0 \
  -device nvme-ns,id=ns1,drive=drv10,nsid=1,subsys=subsys0 \
  -device nvme-ns,id=ns2,drive=drv11,nsid=2,bus=nvme2 \
  \
  -device nvme,serial=qux,id=nvme3 \
  -device nvme-ns,id=ns3,drive=drv12,nsid=3,bus=nvme3 \
  \
  -device nvme-subsys,id=subsys1 \
  -device nvme,serial=quux,id=nvme4,subsys=subsys1 \
  -device nvme-ns,id=ns4,drive=drv13,nsid=1,subsys=subsys1,zoned=true \
  root@vm:~/work# nvme list -v
  NVM Express Subsystems

Subsystem Subsystem-NQN Controllers
---------------- ------------------------------------------------------------------------------------------------ ----------------

  nvme-subsys1     nqn.2019-08.org.qemu:subsys0                                                                     nvme0, nvme1, nvme2
  nvme-subsys3     nqn.2019-08.org.qemu:qux                                                                         nvme3
  nvme-subsys4     nqn.2019-08.org.qemu:subsys1                                                                     nvme4

NVM Express Controllers

Device SN MN FR TxPort Address Subsystem Namespaces
-------- -------------------- ---------------------------------------- -------- ------ -------------- ------------ ----------------

  nvme0    foo                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:06.0   nvme-subsys1 nvme1c0n1
  nvme1    bar                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:07.0   nvme-subsys1 nvme1c1n1
  nvme2    baz                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:08.0   nvme-subsys1 nvme1c2n1, nvme1c2n2
  nvme3    qux                  QEMU NVMe Ctrl                           1.0      pcie   0000:00:09.0   nvme-subsys3 nvme3n1
  nvme4    quux                 QEMU NVMe Ctrl                           1.0      pcie   0000:00:0a.0   nvme-subsys4 nvme4c4n1

NVM Express Namespaces

Device NSID Usage Format Controllers
------------ -------- -------------------------- ---------------- ----------------

  nvme1n1      1        134.22  MB / 134.22  MB    512   B +  0 B   nvme0, nvme1, nvme2
  nvme1n2      2        268.44  MB / 268.44  MB    512   B +  0 B   nvme2
  nvme3n1      3        268.44  MB / 268.44  MB    512   B +  0 B   nvme3
  nvme4n1      1        268.44  MB / 268.44  MB    512   B +  0 B   nvme4

Thanks,

Since V5:
- Fix endianness for 'cntlid' in Identify Controller data structure.

    (Keith)

Since V4:
- Code clean-up to snprintf rather than duplicating it and copy.

    (Keith)

- Documentation for 'subsys' clean-up. (Keith)
- Remove 'cntlid' param from nvme_init_ctrl(). (Keith)
- Put error_propagate() in nvme_realize(). (Keith)

Since RFC V3:
- Exclude 'deatched' scheme from this series. This will be covered in the next series by covering all the ns-related admin commands including ZNS and ns-mgmt. (Niklas)
- Rebased on nvme-next.
- Remove RFC tag from this V4.

Since RFC V2:
- Rebased on nvme-next branch with trivial patches from the previous version(V2) applied. (Klaus)
- Fix enumeration type name convention with NvmeIdNs prefix. (Klaus)
- Put 'cntlid' to NvmeCtrl instance in nvme_init_ctrl() which was missed in V2.
- Added 'detached' parameter to nvme-ns device to decide whether to attach or not to controller(s) in the subsystem. (Klaus)
- Implemented Identify Active Namespace ID List aprt from Identify Allocated Namespace ID List by removing fall-thru statement.

Since RFC V1:
- Updated namespace sharing scheme to be based on nvme-subsys hierarchy.

Minwoo Im (6):
hw/block/nvme: introduce nvme-subsys device hw/block/nvme: support to map controller to a subsystem hw/block/nvme: add CMIC enum value for Identify Controller hw/block/nvme: support for multi-controller in subsystem hw/block/nvme: add NMIC enum value for Identify Namespace hw/block/nvme: support for shared namespace in subsystem

Quoted 12 lines
 hw/block/meson.build   |   2 +-
 hw/block/nvme-ns.c     |  23 +++++++--
 hw/block/nvme-ns.h     |   7 +++
 hw/block/nvme-subsys.c | 106 +++++++++++++++++++++++++++++++++++++++++
 hw/block/nvme-subsys.h |  32 +++++++++++++
 hw/block/nvme.c        |  72 +++++++++++++++++++++++++---
 hw/block/nvme.h        |   4 ++
 include/block/nvme.h   |   8 ++++
 8 files changed, 242 insertions(+), 12 deletions(-)
 create mode 100644 hw/block/nvme-subsys.c
 create mode 100644 hw/block/nvme-subsys.h
Quoted 7 lines

Thanks Minwoo! This really is great stuff.

Notwithstanding the nitpicks I've pointed out for [1/6] and [6/6] (no need to v7 for those), take my

Thanks Klaus!

#9 lore raw

Re: [PATCH V6 4/6] hw/block/nvme: support for multi-controller in subsystem

In reply to Minwoo Im
Quoted 41 lines · Minwoo Im
Quoted 40 lines · Keith Busch
Quoted 38 lines · Klaus Jensen
Quoted 27 lines · Minwoo Im

We have nvme-subsys and nvme devices mapped together. To support multi-controller scheme to this setup, controller identifier(id) has to be managed. Earlier, cntlid(controller id) used to be always 0 because we didn't have any subsystem scheme that controller id matters.

This patch introduced 'cntlid' attribute to the nvme controller instance(NvmeCtrl) and make it allocated by the nvme-subsys device mapped to the controller. If nvme-subsys is not given to the controller, then it will always be 0 as it was.

Added 'ctrls' array in the nvme-subsys instance to manage attached controllers to the subsystem with a limit(32). This patch didn't take list for the controllers to make it seamless with nvme-ns device.

---
 hw/block/nvme-subsys.c | 21 +++++++++++++++++++++
 hw/block/nvme-subsys.h |  4 ++++
 hw/block/nvme.c        | 29 +++++++++++++++++++++++++++++
 hw/block/nvme.h        |  1 +
 4 files changed, 55 insertions(+)
hw/block/nvme.c +0 -0
index b525fca14103..7138389be4bd 100644

hw/block/nvme.c static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) · line 4489

Quoted 6 lines
hw/block/nvme.c +3 -0
@@ -4481,6 +4484,10 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->psd[0].enlat = cpu_to_le32(0x10);
     id->psd[0].exlat = cpu_to_le32(0x4);
 
+    if (n->subsys) {
+        id->cmic |= NVME_CMIC_MULTI_CTRL;
+    }

Since multiple controllers show up with a PCIe port of their own, do we need to set bit 0 (NVME_CMIC_MULTI_PORT?) as well? Or am I misunderstanding that bit?

Quoted 8 lines
Quoted 5 lines

AIUI, if you report this MULTI_PORT bit, then each PCI device in the subsystem needs to report a different "Port Number" in their PCIe Link Capabilities register. I don't think we can manipulate that value from the nvme "device", but I also don't know what a host should do with this information even if we could. So, I think it's safe to leave it at 0.

AFAIK, If we leave it to 0, kernel will not allocate disk for multi-path case (e.g., nvmeXcYnZ).

Kernel only checks for MULTI_CTRL. It doesn't do anything with MULTI_PORT.

author
#10 lore raw

Re: [PATCH V6 4/6] hw/block/nvme: support for multi-controller in subsystem

In reply to Keith Busch
Quoted 48 lines · Keith Busch
Quoted 47 lines · Minwoo Im
Quoted 45 lines · Keith Busch
Quoted 31 lines · Klaus Jensen
Quoted 30 lines · Minwoo Im

We have nvme-subsys and nvme devices mapped together. To support multi-controller scheme to this setup, controller identifier(id) has to be managed. Earlier, cntlid(controller id) used to be always 0 because we didn't have any subsystem scheme that controller id matters.

This patch introduced 'cntlid' attribute to the nvme controller instance(NvmeCtrl) and make it allocated by the nvme-subsys device mapped to the controller. If nvme-subsys is not given to the controller, then it will always be 0 as it was.

Added 'ctrls' array in the nvme-subsys instance to manage attached controllers to the subsystem with a limit(32). This patch didn't take list for the controllers to make it seamless with nvme-ns device.

---
 hw/block/nvme-subsys.c | 21 +++++++++++++++++++++
 hw/block/nvme-subsys.h |  4 ++++
 hw/block/nvme.c        | 29 +++++++++++++++++++++++++++++
 hw/block/nvme.h        |  1 +
 4 files changed, 55 insertions(+)
hw/block/nvme.c +0 -0
index b525fca14103..7138389be4bd 100644
@@ -4481,6 +4484,10 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->psd[0].enlat = cpu_to_le32(0x10);
     id->psd[0].exlat = cpu_to_le32(0x4);
 
Quoted 7 lines

hw/block/nvme.c static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) · line 4489

Quoted 3 lines
hw/block/nvme.c +3 -0
+    if (n->subsys) {
+        id->cmic |= NVME_CMIC_MULTI_CTRL;
+    }

Since multiple controllers show up with a PCIe port of their own, do we need to set bit 0 (NVME_CMIC_MULTI_PORT?) as well? Or am I misunderstanding that bit?

AIUI, if you report this MULTI_PORT bit, then each PCI device in the subsystem needs to report a different "Port Number" in their PCIe Link Capabilities register. I don't think we can manipulate that value from the nvme "device", but I also don't know what a host should do with this information even if we could. So, I think it's safe to leave it at 0.

Quoted 4 lines
Quoted 2 lines

AFAIK, If we leave it to 0, kernel will not allocate disk for multi-path case (e.g., nvmeXcYnZ).

Kernel only checks for MULTI_CTRL. It doesn't do anything with MULTI_PORT.

Please forgive me that I took this discussion as MULTI_CTRL rather than MULTI_PORT. Please ignore this noise ;)

Thanks!

Applied
#18 lore raw
In reply to Minwoo Im
Quoted 9 lines · Minwoo Im

Minwoo Im (6):
hw/block/nvme: introduce nvme-subsys device hw/block/nvme: support to map controller to a subsystem hw/block/nvme: add CMIC enum value for Identify Controller hw/block/nvme: support for multi-controller in subsystem hw/block/nvme: add NMIC enum value for Identify Namespace hw/block/nvme: support for shared namespace in subsystem

Quoted 12 lines
 hw/block/meson.build   |   2 +-
 hw/block/nvme-ns.c     |  23 +++++++--
 hw/block/nvme-ns.h     |   7 +++
 hw/block/nvme-subsys.c | 106 +++++++++++++++++++++++++++++++++++++++++
 hw/block/nvme-subsys.h |  32 +++++++++++++
 hw/block/nvme.c        |  72 +++++++++++++++++++++++++---
 hw/block/nvme.h        |   4 ++
 include/block/nvme.h   |   8 ++++
 8 files changed, 242 insertions(+), 12 deletions(-)
 create mode 100644 hw/block/nvme-subsys.c
 create mode 100644 hw/block/nvme-subsys.h

Thanks Minwoo, this was a super awesome series.

Applied to nvme-next!

#19 lore raw
Quoted 23 lines · Klaus Jensen
Quoted 9 lines · Minwoo Im

Minwoo Im (6):
hw/block/nvme: introduce nvme-subsys device hw/block/nvme: support to map controller to a subsystem hw/block/nvme: add CMIC enum value for Identify Controller hw/block/nvme: support for multi-controller in subsystem hw/block/nvme: add NMIC enum value for Identify Namespace hw/block/nvme: support for shared namespace in subsystem

Quoted 12 lines
 hw/block/meson.build   |   2 +-
 hw/block/nvme-ns.c     |  23 +++++++--
 hw/block/nvme-ns.h     |   7 +++
 hw/block/nvme-subsys.c | 106 +++++++++++++++++++++++++++++++++++++++++
 hw/block/nvme-subsys.h |  32 +++++++++++++
 hw/block/nvme.c        |  72 +++++++++++++++++++++++++---
 hw/block/nvme.h        |   4 ++
 include/block/nvme.h   |   8 ++++
 8 files changed, 242 insertions(+), 12 deletions(-)
 create mode 100644 hw/block/nvme-subsys.c
 create mode 100644 hw/block/nvme-subsys.h
Quoted 3 lines

Thanks Minwoo, this was a super awesome series.

Applied to nvme-next!

By the way, we are really cluttering up hw/block now, so I'm gonna submit a patch based on the one that Philippe did to move the device into hw/nvme.