Skip to contentSkip to conversation

hw/block/nvme: misc cmb/pmr patches and bump to v1.4

PATCH Superseded Klaus Jensen 3 participants 34 messages Jan 18, 2021 – Jan 19, 2021

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover hw/block/nvme: misc cmb/pmr patches and bump to v1.4

From: Klaus Jensen <[email protected]>

This is a resend of "hw/block/nvme: allow cmb and pmr to coexist" with some more PMR work added (PMR RDS/WDS support).

This includes a resurrection of Andrzej's series[1] from back July.

Andrzej's main patch basically moved the CMB from BAR 2 into an offset in BAR 4 (located after the MSI-X table and PBA). Having an offset on the CMB causes a bunch of calculations related to address mapping to change.

So, since I couldn't get the patch to apply cleanly I took a stab at implementing the suggestion I originally came up with: simply move the MSI-X table and PBA from BAR 4 into BAR 0 (up-aligned to a 4 KiB boundary after the main NVMe controller registers). This way we can keep the CMB at offset zero in its own BAR and free up BAR 4 for use by PMR. This makes the patch simpler and does not impact any of the existing address mapping code.

[1]: https://lore.kernel.org/qemu-devel/[email protected]/

Changes for v2
~~~~~~~~~~~~~~

- Rebased on nvme-next
- Added a fix for 64 bit register hi/lo split writes
- Added the patches from "hw/block/nvme: cmb enhancements and bump to v1.4" to the back of this.
- As suggested by Keith, I removed "legacy CMB" support - the patch now exclusively bumps the support to the "v1.4 variant", so the linux kernel nvme gang have to get their game on ;)

Andrzej Jakowski (1):
hw/block/nvme: indicate CMB support through controller capabilities
register

Klaus Jensen (9):
hw/block/nvme: add size to mmio read/write trace events hw/block/nvme: fix 64 bit register hi/lo split writes hw/block/nvme: move msix table and pba to BAR 0 hw/block/nvme: allow cmb and pmr to coexist hw/block/nvme: rename PMR/CMB shift/mask fields hw/block/nvme: remove redundant zeroing of PMR registers hw/block/nvme: disable PMR at boot up hw/block/nvme: bump to v1.4
hw/block/nvme: lift cmb restrictions

Naveen Nagar (1):
hw/block/nvme: add PMR RDS/WDS support

Padmakar Kalghatgi (1):
hw/block/nvme: move cmb logic to v1.4

 hw/block/nvme.h       |   3 +
 include/block/nvme.h  | 125 ++++++++++++++---
 hw/block/nvme.c       | 307 +++++++++++++++++++++++++++---------------
 hw/block/trace-events |   6 +-
 4 files changed, 314 insertions(+), 127 deletions(-)
Signature · 1 line
2.30.0
#1 lore raw

PATCH v2 1/12 hw/block/nvme: add size to mmio read/write trace events

From: Klaus Jensen <[email protected]>

Add the size of the mmio read/write to the trace event.

---
 hw/block/nvme.c       | 4 ++--
 hw/block/trace-events | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
hw/block/nvme.c +2 -2
index 47da73ce2364..bd7e258c3c26 100644
@@ -3840,7 +3840,7 @@ static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
     uint8_t *ptr = (uint8_t *)&n->bar;
     uint64_t val = 0;
 
-    trace_pci_nvme_mmio_read(addr);
+    trace_pci_nvme_mmio_read(addr, size);
 
     if (unlikely(addr & (sizeof(uint32_t) - 1))) {
         NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
@@ -4004,7 +4004,7 @@ static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
 {
     NvmeCtrl *n = (NvmeCtrl *)opaque;
 
-    trace_pci_nvme_mmio_write(addr, data);
+    trace_pci_nvme_mmio_write(addr, data, size);
 
     if (addr < sizeof(n->bar)) {
         nvme_write_bar(n, addr, data, size);
hw/block/trace-events +2 -2
index 78d76b0a71c1..a104d7f4da80 100644
@@ -80,8 +80,8 @@ pci_nvme_enqueue_event_noqueue(int queued) "queued %d"
 pci_nvme_enqueue_event_masked(uint8_t typ) "type 0x%"PRIx8""
 pci_nvme_no_outstanding_aers(void) "ignoring event; no outstanding AERs"
 pci_nvme_enqueue_req_completion(uint16_t cid, uint16_t cqid, uint16_t status) "cid %"PRIu16" cqid %"PRIu16" status 0x%"PRIx16""
-pci_nvme_mmio_read(uint64_t addr) "addr 0x%"PRIx64""
-pci_nvme_mmio_write(uint64_t addr, uint64_t data) "addr 0x%"PRIx64" data 0x%"PRIx64""
+pci_nvme_mmio_read(uint64_t addr, unsigned size) "addr 0x%"PRIx64" size %d"
+pci_nvme_mmio_write(uint64_t addr, uint64_t data, unsigned size) "addr 0x%"PRIx64" data 0x%"PRIx64" size %d"
 pci_nvme_mmio_doorbell_cq(uint16_t cqid, uint16_t new_head) "cqid %"PRIu16" new_head %"PRIu16""
 pci_nvme_mmio_doorbell_sq(uint16_t sqid, uint16_t new_tail) "sqid %"PRIu16" new_tail %"PRIu16""
 pci_nvme_mmio_intm_set(uint64_t data, uint64_t new_mask) "wrote MMIO, interrupt mask set, data=0x%"PRIx64", new_mask=0x%"PRIx64""
Signature · 1 line
2.30.0
#3 lore raw

PATCH v2 2/12 hw/block/nvme: fix 64 bit register hi/lo split writes

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

64 bit registers like ASQ and ACQ should be writable by both a hi/lo 32 bit write combination as well as a plain 64 bit write. The spec does not define ordering on the hi/lo split, but the code currently assumes that the low order bits are written first. Additionally, the code does not consider that another address might already have been written into the register, causing the OR'ing to result in a bad address.

Fix this by explicitly overwriting only the low or high order bits for 32 bit writes.

---
 hw/block/nvme.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
hw/block/nvme.c +6 -4
index bd7e258c3c26..40b9f8ccfc0e 100644
@@ -3781,19 +3781,21 @@ static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
         trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
         break;
     case 0x28:  /* ASQ */
-        n->bar.asq = data;
+        n->bar.asq = size == 8 ? data :
+            (n->bar.asq & ~0xffffffff) | (data & 0xffffffff);
         trace_pci_nvme_mmio_asqaddr(data);
         break;
     case 0x2c:  /* ASQ hi */
-        n->bar.asq |= data << 32;
+        n->bar.asq = (n->bar.asq & 0xffffffff) | (data << 32);
         trace_pci_nvme_mmio_asqaddr_hi(data, n->bar.asq);
         break;
     case 0x30:  /* ACQ */
         trace_pci_nvme_mmio_acqaddr(data);
-        n->bar.acq = data;
+        n->bar.acq = size == 8 ? data :
+            (n->bar.acq & ~0xffffffff) | (data & 0xffffffff);
         break;
     case 0x34:  /* ACQ hi */
-        n->bar.acq |= data << 32;
+        n->bar.acq = (n->bar.acq & 0xffffffff) | (data << 32);
         trace_pci_nvme_mmio_acqaddr_hi(data, n->bar.acq);
         break;
     case 0x38:  /* CMBLOC */
Signature · 1 line
2.30.0
#4 lore raw

Re: [PATCH v2 02/12] hw/block/nvme: fix 64 bit register hi/lo split writes

Quoted 22 lines · Klaus Jensen

From: Klaus Jensen <[email protected]>

64 bit registers like ASQ and ACQ should be writable by both a hi/lo 32 bit write combination as well as a plain 64 bit write. The spec does not define ordering on the hi/lo split, but the code currently assumes that the low order bits are written first. Additionally, the code does not consider that another address might already have been written into the register, causing the OR'ing to result in a bad address.

Fix this by explicitly overwriting only the low or high order bits for 32 bit writes.

---
 hw/block/nvme.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
hw/block/nvme.c +0 -0
index bd7e258c3c26..40b9f8ccfc0e 100644

hw/block/nvme.c static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data, · line 3785

Quoted 6 lines
hw/block/nvme.c +2 -1
@@ -3781,19 +3781,21 @@ static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
         trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
         break;
     case 0x28:  /* ASQ */
-        n->bar.asq = data;
+        n->bar.asq = size == 8 ? data :
+            (n->bar.asq & ~0xffffffff) | (data & 0xffffffff);
                             ^^^^^^^^^^^

If this one is to unmask lower 32bits other than higher 32bits, then it should be:

	(n->bar.asq & ~0xffffffffULL)

Also, maybe we should take care of lower than 4bytes as:

	.min_access_size = 2,
	.max_access_size = 8,

Even we have some error messages up there with:

	if (unlikely(size < sizeof(uint32_t))) {
	    NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
			   "MMIO write smaller than 32-bits,"
			   " offset=0x%"PRIx64", size=%u",
			   offset, size);
	    /* should be ignored, fall through for now */
	}

It's a fall-thru error, and that's it. I would prefer not to have this error and support for 2bytes access also, OR do not support for 2bytes access for this MMIO area.

Thanks!

#5 lore raw

Re: [PATCH v2 02/12] hw/block/nvme: fix 64 bit register hi/lo split writes

Quoted 30 lines · Minwoo Im
Quoted 22 lines · Klaus Jensen

From: Klaus Jensen <[email protected]>

64 bit registers like ASQ and ACQ should be writable by both a hi/lo 32 bit write combination as well as a plain 64 bit write. The spec does not define ordering on the hi/lo split, but the code currently assumes that the low order bits are written first. Additionally, the code does not consider that another address might already have been written into the register, causing the OR'ing to result in a bad address.

Fix this by explicitly overwriting only the low or high order bits for 32 bit writes.

---
 hw/block/nvme.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
hw/block/nvme.c +0 -0
index bd7e258c3c26..40b9f8ccfc0e 100644

hw/block/nvme.c static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data, · line 3785

Quoted 6 lines
hw/block/nvme.c +2 -1
@@ -3781,19 +3781,21 @@ static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
         trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
         break;
     case 0x28:  /* ASQ */
-        n->bar.asq = data;
+        n->bar.asq = size == 8 ? data :
+            (n->bar.asq & ~0xffffffff) | (data & 0xffffffff);
                             ^^^^^^^^^^^
Quoted 5 lines

If this one is to unmask lower 32bits other than higher 32bits, then it should be:

	(n->bar.asq & ~0xffffffffULL)

Ouch. Yes, thanks!

Quoted 15 lines

Also, maybe we should take care of lower than 4bytes as:

	.min_access_size = 2,
	.max_access_size = 8,

Even we have some error messages up there with:

	if (unlikely(size < sizeof(uint32_t))) {
	    NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
			   "MMIO write smaller than 32-bits,"
			   " offset=0x%"PRIx64", size=%u",
			   offset, size);
	    /* should be ignored, fall through for now */
	}
Quoted 4 lines

It's a fall-thru error, and that's it. I would prefer not to have this error and support for 2bytes access also, OR do not support for 2bytes access for this MMIO area.

The spec requires aligned 32 bit accesses (or the size of the register), so maybe it's about time we actually ignore instead of falling through.

#6 lore raw

Re: [PATCH v2 02/12] hw/block/nvme: fix 64 bit register hi/lo split writes

Quoted 2 lines

The spec requires aligned 32 bit accesses (or the size of the register), so maybe it's about time we actually ignore instead of falling through.

Agreed.

#7 lore raw

Re: [PATCH v2 02/12] hw/block/nvme: fix 64 bit register hi/lo split writes

Quoted 5 lines · Minwoo Im
Quoted 2 lines

The spec requires aligned 32 bit accesses (or the size of the register), so maybe it's about time we actually ignore instead of falling through.

Agreed.

On the other hand.

The spec just allows undefined behavior if the alignment or size requirement is violated. So falling through is not wrong.

Keith, any thoughts on this?

#8 lore raw

Re: [PATCH v2 02/12] hw/block/nvme: fix 64 bit register hi/lo split writes

Quoted 11 lines · Klaus Jensen
Quoted 5 lines · Minwoo Im
Quoted 2 lines

The spec requires aligned 32 bit accesses (or the size of the register), so maybe it's about time we actually ignore instead of falling through.

Agreed.

On the other hand.

The spec just allows undefined behavior if the alignment or size requirement is violated. So falling through is not wrong.

If so, maybe we just can make this MMIO region support under 4bytes access with error messaging. I don't think we should not support them on purpose because spec says it just results in undefined behaviors which is just undefined how to handle them.

#9 lore raw

Re: [PATCH v2 02/12] hw/block/nvme: fix 64 bit register hi/lo split writes

In reply to Klaus Jensen
Quoted 7 lines · Klaus Jensen
Quoted 5 lines · Minwoo Im
Quoted 2 lines

The spec requires aligned 32 bit accesses (or the size of the register), so maybe it's about time we actually ignore instead of falling through.

Agreed.

Quoted 6 lines

On the other hand.

The spec just allows undefined behavior if the alignment or size requirement is violated. So falling through is not wrong.

Keith, any thoughts on this?

If the host's access is unaligned and breaks something, the host gets to keep both pieces.

#10 lore raw

PATCH v2 3/12 hw/block/nvme: indicate CMB support through controller capabilities register

In reply to Klaus Jensen

From: Andrzej Jakowski <[email protected]>

This patch sets CMBS bit in controller capabilities register when user configures NVMe driver with CMB support, so capabilites are correctly reported to guest OS.

---
 include/block/nvme.h | 10 +++++++---
 hw/block/nvme.c      |  1 +
 2 files changed, 8 insertions(+), 3 deletions(-)
include/block/nvme.h +7 -3
index 45b2678db1f0..86d7fc2f905c 100644
@@ -36,6 +36,7 @@ enum NvmeCapShift {
     CAP_MPSMIN_SHIFT   = 48,
     CAP_MPSMAX_SHIFT   = 52,
     CAP_PMR_SHIFT      = 56,
+    CAP_CMB_SHIFT      = 57,
 };
 
 enum NvmeCapMask {
@@ -49,6 +50,7 @@ enum NvmeCapMask {
     CAP_MPSMIN_MASK    = 0xf,
     CAP_MPSMAX_MASK    = 0xf,
     CAP_PMR_MASK       = 0x1,
+    CAP_CMB_MASK       = 0x1,
 };
 
 #define NVME_CAP_MQES(cap)  (((cap) >> CAP_MQES_SHIFT)   & CAP_MQES_MASK)
@@ -78,9 +80,11 @@ enum NvmeCapMask {
 #define NVME_CAP_SET_MPSMIN(cap, val) (cap |= (uint64_t)(val & CAP_MPSMIN_MASK)\
                                                            << CAP_MPSMIN_SHIFT)
 #define NVME_CAP_SET_MPSMAX(cap, val) (cap |= (uint64_t)(val & CAP_MPSMAX_MASK)\
-                                                            << CAP_MPSMAX_SHIFT)
-#define NVME_CAP_SET_PMRS(cap, val) (cap |= (uint64_t)(val & CAP_PMR_MASK)\
-                                                            << CAP_PMR_SHIFT)
+                                                           << CAP_MPSMAX_SHIFT)
+#define NVME_CAP_SET_PMRS(cap, val)   (cap |= (uint64_t)(val & CAP_PMR_MASK)   \
+                                                           << CAP_PMR_SHIFT)
+#define NVME_CAP_SET_CMBS(cap, val)   (cap |= (uint64_t)(val & CAP_CMB_MASK)   \
+                                                           << CAP_CMB_SHIFT)
 
 enum NvmeCapCss {
     NVME_CAP_CSS_NVM        = 1 << 0,
hw/block/nvme.c +1 -0
index 40b9f8ccfc0e..606006c549bc 100644
@@ -4336,6 +4336,7 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     NVME_CAP_SET_CSS(n->bar.cap, NVME_CAP_CSS_CSI_SUPP);
     NVME_CAP_SET_CSS(n->bar.cap, NVME_CAP_CSS_ADMIN_ONLY);
     NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
+    NVME_CAP_SET_CMBS(n->bar.cap, n->params.cmb_size_mb ? 1 : 0);
 
     n->bar.vs = NVME_SPEC_VER;
     n->bar.intmc = n->bar.intms = 0;
Signature · 1 line
2.30.0
#12 lore raw

PATCH v2 4/12 hw/block/nvme: move msix table and pba to BAR 0

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

In the interest of supporting both CMB and PMR to be enabled on the same device, move the MSI-X table and pending bit array out of BAR 4 and into BAR 0.

This is a simplified version of the patch contributed by Andrzej Jakowski (see [1]). Leaving the CMB at offset 0 removes the need for changes to CMB address mapping code.

[1]: https://lore.kernel.org/qemu-devel/[email protected]/

---
 hw/block/nvme.h |  1 +
 hw/block/nvme.c | 23 +++++++++++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)
hw/block/nvme.h +1 -0
index 65540b650e1d..2a25bc84f3f9 100644
@@ -126,6 +126,7 @@ typedef struct NvmeFeatureVal {
 
 typedef struct NvmeCtrl {
     PCIDevice    parent_obj;
+    MemoryRegion bar0;
     MemoryRegion iomem;
     MemoryRegion ctrl_mem;
     NvmeBar      bar;
hw/block/nvme.c +21 -2
index 606006c549bc..ec2104fcf3b6 100644
@@ -4230,6 +4230,8 @@ static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
 static int nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
 {
     uint8_t *pci_conf = pci_dev->config;
+    uint64_t bar_size, msix_table_size, msix_pba_size;
+    unsigned msix_table_offset, msix_pba_offset;
     int ret;
 
     Error *err = NULL;
@@ -4248,11 +4250,28 @@ static int nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
     pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
     pcie_endpoint_cap_init(pci_dev, 0x80);
 
+    bar_size = QEMU_ALIGN_UP(n->reg_size, 4 * KiB);
+    msix_table_offset = bar_size;
+    msix_table_size = PCI_MSIX_ENTRY_SIZE * n->params.msix_qsize;
+
+    bar_size += msix_table_size;
+    bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
+    msix_pba_offset = bar_size;
+    msix_pba_size = QEMU_ALIGN_UP(n->params.msix_qsize, 64) / 8;
+
+    bar_size += msix_pba_size;
+    bar_size = pow2ceil(bar_size);
+
+    memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
     memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
                           n->reg_size);
+    memory_region_add_subregion(&n->bar0, 0, &n->iomem);
+
     pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
-                     PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem);
-    ret = msix_init_exclusive_bar(pci_dev, n->params.msix_qsize, 4, &err);
+                     PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
+    ret = msix_init(pci_dev, n->params.msix_qsize,
+                    &n->bar0, 0, msix_table_offset,
+                    &n->bar0, 0, msix_pba_offset, 0, &err);
     if (ret < 0) {
         if (ret == -ENOTSUP) {
             warn_report_err(err);
Signature · 1 line
2.30.0
#14 lore raw

PATCH v2 5/12 hw/block/nvme: allow cmb and pmr to coexist

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

With BAR 4 now free to use, allow PMR and CMB to be enabled simultaneously.

---
 hw/block/nvme.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
hw/block/nvme.c +8 -9
index ec2104fcf3b6..f3bea582b3c0 100644
@@ -29,14 +29,13 @@
  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
  * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
  *
- * cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation
- * in available BAR's. cmb_size_mb= will take precedence over pmrdev= when
- * both provided.
  * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
  * For example:
  * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
  *  size=<size> .... -device nvme,...,pmrdev=<mem_id>
  *
+ * The PMR will use BAR 4/5 exclusively.
+ *
  *
  * nvme device parameters
  * ~~~~~~~~~~~~~~~~~~~~~~
@@ -109,7 +108,7 @@
 #define NVME_DB_SIZE  4
 #define NVME_SPEC_VER 0x00010300
 #define NVME_CMB_BIR 2
-#define NVME_PMR_BIR 2
+#define NVME_PMR_BIR 4
 #define NVME_TEMPERATURE 0x143
 #define NVME_TEMPERATURE_WARNING 0x157
 #define NVME_TEMPERATURE_CRITICAL 0x175
@@ -4083,7 +4082,7 @@ static void nvme_check_constraints(NvmeCtrl *n, Error **errp)
         return;
     }
 
-    if (!n->params.cmb_size_mb && n->pmrdev) {
+    if (n->pmrdev) {
         if (host_memory_backend_is_mapped(n->pmrdev)) {
             error_setg(errp, "can't use already busy memdev: %s",
                        object_get_canonical_path_component(OBJECT(n->pmrdev)));
@@ -4180,9 +4179,6 @@ static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
 
 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
 {
-    /* Controller Capabilities register */
-    NVME_CAP_SET_PMRS(n->bar.cap, 1);
-
     /* PMR Capabities register */
     n->bar.pmrcap = 0;
     NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
@@ -4283,7 +4279,9 @@ static int nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
 
     if (n->params.cmb_size_mb) {
         nvme_init_cmb(n, pci_dev);
-    } else if (n->pmrdev) {
+    }
+
+    if (n->pmrdev) {
         nvme_init_pmr(n, pci_dev);
     }
 
@@ -4356,6 +4354,7 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     NVME_CAP_SET_CSS(n->bar.cap, NVME_CAP_CSS_ADMIN_ONLY);
     NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
     NVME_CAP_SET_CMBS(n->bar.cap, n->params.cmb_size_mb ? 1 : 0);
+    NVME_CAP_SET_PMRS(n->bar.cap, n->pmrdev ? 1 : 0);
 
     n->bar.vs = NVME_SPEC_VER;
     n->bar.intmc = n->bar.intms = 0;
Signature · 1 line
2.30.0
#16 lore raw

PATCH v2 6/12 hw/block/nvme: rename PMR/CMB shift/mask fields

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

Use the correct field names.

---
 include/block/nvme.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
include/block/nvme.h +8 -8
index 86d7fc2f905c..f3cbe17d0971 100644
@@ -35,8 +35,8 @@ enum NvmeCapShift {
     CAP_CSS_SHIFT      = 37,
     CAP_MPSMIN_SHIFT   = 48,
     CAP_MPSMAX_SHIFT   = 52,
-    CAP_PMR_SHIFT      = 56,
-    CAP_CMB_SHIFT      = 57,
+    CAP_PMRS_SHIFT     = 56,
+    CAP_CMBS_SHIFT     = 57,
 };
 
 enum NvmeCapMask {
@@ -49,8 +49,8 @@ enum NvmeCapMask {
     CAP_CSS_MASK       = 0xff,
     CAP_MPSMIN_MASK    = 0xf,
     CAP_MPSMAX_MASK    = 0xf,
-    CAP_PMR_MASK       = 0x1,
-    CAP_CMB_MASK       = 0x1,
+    CAP_PMRS_MASK      = 0x1,
+    CAP_CMBS_MASK      = 0x1,
 };
 
 #define NVME_CAP_MQES(cap)  (((cap) >> CAP_MQES_SHIFT)   & CAP_MQES_MASK)
@@ -81,10 +81,10 @@ enum NvmeCapMask {
                                                            << CAP_MPSMIN_SHIFT)
 #define NVME_CAP_SET_MPSMAX(cap, val) (cap |= (uint64_t)(val & CAP_MPSMAX_MASK)\
                                                            << CAP_MPSMAX_SHIFT)
-#define NVME_CAP_SET_PMRS(cap, val)   (cap |= (uint64_t)(val & CAP_PMR_MASK)   \
-                                                           << CAP_PMR_SHIFT)
-#define NVME_CAP_SET_CMBS(cap, val)   (cap |= (uint64_t)(val & CAP_CMB_MASK)   \
-                                                           << CAP_CMB_SHIFT)
+#define NVME_CAP_SET_PMRS(cap, val)   (cap |= (uint64_t)(val & CAP_PMRS_MASK)  \
+                                                           << CAP_PMRS_SHIFT)
+#define NVME_CAP_SET_CMBS(cap, val)   (cap |= (uint64_t)(val & CAP_CMBS_MASK)  \
+                                                           << CAP_CMBS_SHIFT)
 
 enum NvmeCapCss {
     NVME_CAP_CSS_NVM        = 1 << 0,
Signature · 1 line
2.30.0
Reviewed-by on v2 6/12
#17 lore raw

Re: [PATCH v2 06/12] hw/block/nvme: rename PMR/CMB shift/mask fields

Quoted 37 lines · Klaus Jensen

From: Klaus Jensen <[email protected]>

Use the correct field names.

---
 include/block/nvme.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
include/block/nvme.h +4 -4
index 86d7fc2f905c..f3cbe17d0971 100644
@@ -35,8 +35,8 @@ enum NvmeCapShift {
     CAP_CSS_SHIFT      = 37,
     CAP_MPSMIN_SHIFT   = 48,
     CAP_MPSMAX_SHIFT   = 52,
-    CAP_PMR_SHIFT      = 56,
-    CAP_CMB_SHIFT      = 57,
+    CAP_PMRS_SHIFT     = 56,
+    CAP_CMBS_SHIFT     = 57,
 };
 
 enum NvmeCapMask {
@@ -49,8 +49,8 @@ enum NvmeCapMask {
     CAP_CSS_MASK       = 0xff,
     CAP_MPSMIN_MASK    = 0xf,
     CAP_MPSMAX_MASK    = 0xf,
-    CAP_PMR_MASK       = 0x1,
-    CAP_CMB_MASK       = 0x1,
+    CAP_PMRS_MASK      = 0x1,
+    CAP_CMBS_MASK      = 0x1,
 };
 
 #define NVME_CAP_MQES(cap)  (((cap) >> CAP_MQES_SHIFT)   & CAP_MQES_MASK)
@@ -81,10 +81,10 @@ enum NvmeCapMask {
                                                            << CAP_MPSMIN_SHIFT)

include/block/nvme.h enum NvmeCapMask { · line 87

Quoted 10 lines
include/block/nvme.h +4 -4
 #define NVME_CAP_SET_MPSMAX(cap, val) (cap |= (uint64_t)(val & CAP_MPSMAX_MASK)\
                                                            << CAP_MPSMAX_SHIFT)
-#define NVME_CAP_SET_PMRS(cap, val)   (cap |= (uint64_t)(val & CAP_PMR_MASK)   \
-                                                           << CAP_PMR_SHIFT)
-#define NVME_CAP_SET_CMBS(cap, val)   (cap |= (uint64_t)(val & CAP_CMB_MASK)   \
-                                                           << CAP_CMB_SHIFT)
+#define NVME_CAP_SET_PMRS(cap, val)   (cap |= (uint64_t)(val & CAP_PMRS_MASK)  \
+                                                           << CAP_PMRS_SHIFT)
+#define NVME_CAP_SET_CMBS(cap, val)   (cap |= (uint64_t)(val & CAP_CMBS_MASK)  \
+                                                           << CAP_CMBS_SHIFT)

Oh, it would have been better folded into [3/12] patch, though.

Changes are looking good to me to represent "Supported".

Quoted 7 lines
include/block/nvme.h +0 -0
 enum NvmeCapCss {
     NVME_CAP_CSS_NVM        = 1 << 0,
Signature · 1 line
2.30.0
#18 lore raw

PATCH v2 7/12 hw/block/nvme: remove redundant zeroing of PMR registers

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

The controller registers are initially zero. Remove the redundant zeroing.

---
 hw/block/nvme.c | 35 -----------------------------------
 1 file changed, 35 deletions(-)
hw/block/nvme.c +0 -35
index f3bea582b3c0..9ee9570bb65c 100644
@@ -4179,43 +4179,8 @@ static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
 
 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
 {
-    /* PMR Capabities register */
-    n->bar.pmrcap = 0;
-    NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
-    NVME_PMRCAP_SET_WDS(n->bar.pmrcap, 0);
     NVME_PMRCAP_SET_BIR(n->bar.pmrcap, NVME_PMR_BIR);
-    NVME_PMRCAP_SET_PMRTU(n->bar.pmrcap, 0);
-    /* Turn on bit 1 support */
     NVME_PMRCAP_SET_PMRWBM(n->bar.pmrcap, 0x02);
-    NVME_PMRCAP_SET_PMRTO(n->bar.pmrcap, 0);
-    NVME_PMRCAP_SET_CMSS(n->bar.pmrcap, 0);
-
-    /* PMR Control register */
-    n->bar.pmrctl = 0;
-    NVME_PMRCTL_SET_EN(n->bar.pmrctl, 0);
-
-    /* PMR Status register */
-    n->bar.pmrsts = 0;
-    NVME_PMRSTS_SET_ERR(n->bar.pmrsts, 0);
-    NVME_PMRSTS_SET_NRDY(n->bar.pmrsts, 0);
-    NVME_PMRSTS_SET_HSTS(n->bar.pmrsts, 0);
-    NVME_PMRSTS_SET_CBAI(n->bar.pmrsts, 0);
-
-    /* PMR Elasticity Buffer Size register */
-    n->bar.pmrebs = 0;
-    NVME_PMREBS_SET_PMRSZU(n->bar.pmrebs, 0);
-    NVME_PMREBS_SET_RBB(n->bar.pmrebs, 0);
-    NVME_PMREBS_SET_PMRWBZ(n->bar.pmrebs, 0);
-
-    /* PMR Sustained Write Throughput register */
-    n->bar.pmrswtp = 0;
-    NVME_PMRSWTP_SET_PMRSWTU(n->bar.pmrswtp, 0);
-    NVME_PMRSWTP_SET_PMRSWTV(n->bar.pmrswtp, 0);
-
-    /* PMR Memory Space Control register */
-    n->bar.pmrmsc = 0;
-    NVME_PMRMSC_SET_CMSE(n->bar.pmrmsc, 0);
-    NVME_PMRMSC_SET_CBA(n->bar.pmrmsc, 0);
 
     pci_register_bar(pci_dev, NVME_PMRCAP_BIR(n->bar.pmrcap),
                      PCI_BASE_ADDRESS_SPACE_MEMORY |
Signature · 1 line
2.30.0
#19 lore raw

Re: [PATCH v2 07/12] hw/block/nvme: remove redundant zeroing of PMR registers

Quoted 15 lines · Klaus Jensen

From: Klaus Jensen <[email protected]>

The controller registers are initially zero. Remove the redundant zeroing.

---
 hw/block/nvme.c | 35 -----------------------------------
 1 file changed, 35 deletions(-)
hw/block/nvme.c +0 -0
index f3bea582b3c0..9ee9570bb65c 100644

hw/block/nvme.c static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev) · line 4188

Quoted 10 lines
hw/block/nvme.c +0 -6
@@ -4179,43 +4179,8 @@ static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
 
 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
 {
-    /* PMR Capabities register */
-    n->bar.pmrcap = 0;
-    NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
-    NVME_PMRCAP_SET_WDS(n->bar.pmrcap, 0);
     NVME_PMRCAP_SET_BIR(n->bar.pmrcap, NVME_PMR_BIR);
-    NVME_PMRCAP_SET_PMRTU(n->bar.pmrcap, 0);
-    /* Turn on bit 1 support */

This comment says that PMRWBM [1]th bit is set to PMRCAP below :).

Quoted 37 lines
hw/block/nvme.c +0 -29
     NVME_PMRCAP_SET_PMRWBM(n->bar.pmrcap, 0x02);
-    NVME_PMRCAP_SET_PMRTO(n->bar.pmrcap, 0);
-    NVME_PMRCAP_SET_CMSS(n->bar.pmrcap, 0);
-
-    /* PMR Control register */
-    n->bar.pmrctl = 0;
-    NVME_PMRCTL_SET_EN(n->bar.pmrctl, 0);
-
-    /* PMR Status register */
-    n->bar.pmrsts = 0;
-    NVME_PMRSTS_SET_ERR(n->bar.pmrsts, 0);
-    NVME_PMRSTS_SET_NRDY(n->bar.pmrsts, 0);
-    NVME_PMRSTS_SET_HSTS(n->bar.pmrsts, 0);
-    NVME_PMRSTS_SET_CBAI(n->bar.pmrsts, 0);
-
-    /* PMR Elasticity Buffer Size register */
-    n->bar.pmrebs = 0;
-    NVME_PMREBS_SET_PMRSZU(n->bar.pmrebs, 0);
-    NVME_PMREBS_SET_RBB(n->bar.pmrebs, 0);
-    NVME_PMREBS_SET_PMRWBZ(n->bar.pmrebs, 0);
-
-    /* PMR Sustained Write Throughput register */
-    n->bar.pmrswtp = 0;
-    NVME_PMRSWTP_SET_PMRSWTU(n->bar.pmrswtp, 0);
-    NVME_PMRSWTP_SET_PMRSWTV(n->bar.pmrswtp, 0);
-
-    /* PMR Memory Space Control register */
-    n->bar.pmrmsc = 0;
-    NVME_PMRMSC_SET_CMSE(n->bar.pmrmsc, 0);
-    NVME_PMRMSC_SET_CBA(n->bar.pmrmsc, 0);
 
     pci_register_bar(pci_dev, NVME_PMRCAP_BIR(n->bar.pmrcap),
                      PCI_BASE_ADDRESS_SPACE_MEMORY |
Signature · 1 line
2.30.0
#20 lore raw

Re: [PATCH v2 07/12] hw/block/nvme: remove redundant zeroing of PMR registers

Quoted 17 lines · Minwoo Im
Quoted 16 lines · Klaus Jensen

From: Klaus Jensen <[email protected]>

The controller registers are initially zero. Remove the redundant zeroing.

---
 hw/block/nvme.c | 35 -----------------------------------
 1 file changed, 35 deletions(-)
hw/block/nvme.c +0 -0
index f3bea582b3c0..9ee9570bb65c 100644
@@ -4179,43 +4179,8 @@ static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
 
Quoted 12 lines

hw/block/nvme.c static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev) · line 4188

Quoted 9 lines
hw/block/nvme.c +0 -6
 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
 {
-    /* PMR Capabities register */
-    n->bar.pmrcap = 0;
-    NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
-    NVME_PMRCAP_SET_WDS(n->bar.pmrcap, 0);
     NVME_PMRCAP_SET_BIR(n->bar.pmrcap, NVME_PMR_BIR);
-    NVME_PMRCAP_SET_PMRTU(n->bar.pmrcap, 0);
-    /* Turn on bit 1 support */

This comment says that PMRWBM [1]th bit is set to PMRCAP below :).

Thanks!

#21 lore raw

PATCH v2 8/12 hw/block/nvme: disable PMR at boot up

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

The PMR should not be enabled at boot up. Disable the PMR MemoryRegion initially and implement MMIO for PMRCTL, allowing the host to enable the PMR explicitly.

---
 hw/block/nvme.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)
hw/block/nvme.c +12 -2
index 9ee9570bb65c..937a6ed0a70d 100644
@@ -3810,8 +3810,16 @@ static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
                        "invalid write to PMRCAP register, ignored");
         return;
-    case 0xE04: /* TODO PMRCTL */
-        break;
+    case 0xE04: /* PMRCTL */
+        n->bar.pmrctl = data;
+        if (NVME_PMRCTL_EN(data)) {
+            memory_region_set_enabled(&n->pmrdev->mr, true);
+            n->bar.pmrsts = 0;
+        } else {
+            memory_region_set_enabled(&n->pmrdev->mr, false);
+            NVME_PMRSTS_SET_NRDY(n->bar.pmrsts, 1);
+        }
+        return;
     case 0xE08: /* PMRSTS */
         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
                        "invalid write to PMRSTS register, ignored");
@@ -4186,6 +4194,8 @@ static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
                      PCI_BASE_ADDRESS_SPACE_MEMORY |
                      PCI_BASE_ADDRESS_MEM_TYPE_64 |
                      PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmrdev->mr);
+
+    memory_region_set_enabled(&n->pmrdev->mr, false);
 }
 
 static int nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
Signature · 1 line
2.30.0
#22 lore raw

PATCH v2 9/12 hw/block/nvme: add PMR RDS/WDS support

In reply to Klaus Jensen

From: Naveen Nagar <[email protected]>

Add support for the PMRMSCL and PMRMSCU MMIO registers. This allows adding RDS/WDS support for PMR as well.

---
 hw/block/nvme.h      |  1 +
 include/block/nvme.h |  1 +
 hw/block/nvme.c      | 89 ++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 87 insertions(+), 4 deletions(-)
hw/block/nvme.h +1 -0
index 2a25bc84f3f9..5988d9b36e12 100644
@@ -152,6 +152,7 @@ typedef struct NvmeCtrl {
     uint16_t    temperature;
 
     HostMemoryBackend *pmrdev;
+    bool              pmr_cmse;
 
     uint8_t     aer_mask;
     NvmeRequest **aer_reqs;
include/block/nvme.h +1 -0
index f3cbe17d0971..183dc5c0ecf6 100644
@@ -62,6 +62,7 @@ enum NvmeCapMask {
 #define NVME_CAP_CSS(cap)   (((cap) >> CAP_CSS_SHIFT)    & CAP_CSS_MASK)
 #define NVME_CAP_MPSMIN(cap)(((cap) >> CAP_MPSMIN_SHIFT) & CAP_MPSMIN_MASK)
 #define NVME_CAP_MPSMAX(cap)(((cap) >> CAP_MPSMAX_SHIFT) & CAP_MPSMAX_MASK)
+#define NVME_CAP_PMRS(cap)  (((cap) >> CAP_PMRS_SHIFT)   & CAP_PMRS_MASK)
 
 #define NVME_CAP_SET_MQES(cap, val)   (cap |= (uint64_t)(val & CAP_MQES_MASK)  \
                                                            << CAP_MQES_SHIFT)
hw/block/nvme.c +85 -4

148 lines not on this page · raw message

Signature · 1 line
2.30.0
#23 lore raw

PATCH v2 10/12 hw/block/nvme: move cmb logic to v1.4

In reply to Klaus Jensen

From: Padmakar Kalghatgi <[email protected]>

Implement v1.4 logic for configuring the Controller Memory Buffer. This is not backward compatible with v1.3, so drivers that only support v1.3 will not be able to use the CMB anymore.

---
 hw/block/nvme.h       |   1 +
 include/block/nvme.h  | 107 +++++++++++++++++++++++++++++++++++++-----
 hw/block/nvme.c       |  78 +++++++++++++++++++++++-------
 hw/block/trace-events |   2 +
 4 files changed, 159 insertions(+), 29 deletions(-)
hw/block/nvme.h +1 -0
index 5988d9b36e12..de9164dd52e6 100644
@@ -145,6 +145,7 @@ typedef struct NvmeCtrl {
     uint32_t    max_q_ents;
     uint8_t     outstanding_aers;
     uint8_t     *cmbuf;
+    bool        cmb_cmse;
     uint32_t    irq_status;
     uint64_t    host_timestamp;                 /* Timestamp sent by the host */
     uint64_t    timestamp_set_qemu_clock_ms;    /* QEMU clock time */
include/block/nvme.h +96 -11

147 lines not on this page · raw message

hw/block/nvme.c +60 -18

113 lines not on this page · raw message

hw/block/trace-events +2 -0
index a104d7f4da80..8e249ea910aa 100644
@@ -123,6 +123,8 @@ pci_nvme_err_invalid_opc(uint8_t opc) "invalid opcode 0x%"PRIx8""
 pci_nvme_err_invalid_admin_opc(uint8_t opc) "invalid admin opcode 0x%"PRIx8""
 pci_nvme_err_invalid_lba_range(uint64_t start, uint64_t len, uint64_t limit) "Invalid LBA start=%"PRIu64" len=%"PRIu64" limit=%"PRIu64""
 pci_nvme_err_invalid_log_page_offset(uint64_t ofs, uint64_t size) "must be <= %"PRIu64", got %"PRIu64""
+pci_nvme_err_cmb_invalid_cba(uint64_t cmbmsc) "cmbmsc 0x%"PRIx64""
+pci_nvme_err_cmb_not_enabled(uint64_t cmbmsc) "cmbmsc 0x%"PRIx64""
 pci_nvme_err_unaligned_zone_cmd(uint8_t action, uint64_t slba, uint64_t zslba) "unaligned zone op 0x%"PRIx32", got slba=%"PRIu64", zslba=%"PRIu64""
 pci_nvme_err_invalid_zone_state_transition(uint8_t action, uint64_t slba, uint8_t attrs) "action=0x%"PRIx8", slba=%"PRIu64", attrs=0x%"PRIx32""
 pci_nvme_err_write_not_at_wp(uint64_t slba, uint64_t zone, uint64_t wp) "writing at slba=%"PRIu64", zone=%"PRIu64", but wp=%"PRIu64""
Signature · 1 line
2.30.0
#24 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Hello,

Quoted 8 lines · Klaus Jensen

From: Padmakar Kalghatgi <[email protected]>

Implement v1.4 logic for configuring the Controller Memory Buffer. This is not backward compatible with v1.3, so drivers that only support v1.3 will not be able to use the CMB anymore.

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

#25 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Quoted 9 lines · Minwoo Im

Hello,

Quoted 6 lines · Klaus Jensen

From: Padmakar Kalghatgi <[email protected]>

Implement v1.4 logic for configuring the Controller Memory Buffer. This is not backward compatible with v1.3, so drivers that only support v1.3 will not be able to use the CMB anymore.

Quoted 6 lines
Quoted 2 lines

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

Next patch moves to v1.4. I wanted to split it because the "bump" patch also adds a couple of other v1.4 requires fields. But I understand that this is slightly wrong.

#26 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Quoted 7 lines
Quoted 3 lines

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

Next patch moves to v1.4. I wanted to split it because the "bump" patch also adds a couple of other v1.4 requires fields. But I understand that this is slightly wrong.

Sorry, I meant I'd like to have CMB for v1.3 support along with the v1.4 support in this device model separately. Maybe I missed the linux-nvme mailing list for CMB v1.4, but is there no plan to keep supportin CMB v1.3 in NVMe driver?

#27 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Quoted 4 lines · Minwoo Im
Quoted 4 lines
Quoted 3 lines

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

Quoted 8 lines
Quoted 3 lines

Next patch moves to v1.4. I wanted to split it because the "bump" patch also adds a couple of other v1.4 requires fields. But I understand that this is slightly wrong.

Sorry, I meant I'd like to have CMB for v1.3 support along with the v1.4 support in this device model separately. Maybe I missed the linux-nvme mailing list for CMB v1.4, but is there no plan to keep supportin CMB v1.3 in NVMe driver?

I posted a patch on linux-nvme for v1.4 support in the kernel. It will support both the v1.3 and v1.4 behavior :)

#28 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Quoted 9 lines · Klaus Jensen
Quoted 8 lines · Minwoo Im
Quoted 7 lines
Quoted 3 lines

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

Next patch moves to v1.4. I wanted to split it because the "bump" patch also adds a couple of other v1.4 requires fields. But I understand that this is slightly wrong.

Quoted 7 lines
Quoted 4 lines

Sorry, I meant I'd like to have CMB for v1.3 support along with the v1.4 support in this device model separately. Maybe I missed the linux-nvme mailing list for CMB v1.4, but is there no plan to keep supportin CMB v1.3 in NVMe driver?

I posted a patch on linux-nvme for v1.4 support in the kernel. It will support both the v1.3 and v1.4 behavior :)

Then, can we maintain CMB v1.3 also in QEMU also along with v1.4 ? :)

#29 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Quoted 15 lines · Minwoo Im
Quoted 14 lines · Klaus Jensen
Quoted 12 lines · Minwoo Im
Quoted 7 lines
Quoted 3 lines

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

Next patch moves to v1.4. I wanted to split it because the "bump" patch also adds a couple of other v1.4 requires fields. But I understand that this is slightly wrong.

Sorry, I meant I'd like to have CMB for v1.3 support along with the v1.4 support in this device model separately. Maybe I missed the linux-nvme mailing list for CMB v1.4, but is there no plan to keep supportin CMB v1.3 in NVMe driver?

Quoted 4 lines
Quoted 2 lines

I posted a patch on linux-nvme for v1.4 support in the kernel. It will support both the v1.3 and v1.4 behavior :)

Then, can we maintain CMB v1.3 also in QEMU also along with v1.4 ? :)

My first version of this patch actually included compatibility support for v1.3 ("legacy cmb"), but Keith suggested we just drop that and keep this device compliant.

What we could do is allow the spec version to be chosen with a parameter?

#30 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Quoted 19 lines · Klaus Jensen
Quoted 18 lines · Minwoo Im
Quoted 16 lines · Klaus Jensen
Quoted 4 lines · Minwoo Im
Quoted 4 lines
Quoted 3 lines

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

Quoted 8 lines
Quoted 3 lines

Next patch moves to v1.4. I wanted to split it because the "bump" patch also adds a couple of other v1.4 requires fields. But I understand that this is slightly wrong.

Sorry, I meant I'd like to have CMB for v1.3 support along with the v1.4 support in this device model separately. Maybe I missed the linux-nvme mailing list for CMB v1.4, but is there no plan to keep supportin CMB v1.3 in NVMe driver?

I posted a patch on linux-nvme for v1.4 support in the kernel. It will support both the v1.3 and v1.4 behavior :)

Quoted 8 lines
Quoted 1 line

Then, can we maintain CMB v1.3 also in QEMU also along with v1.4 ? :)

My first version of this patch actually included compatibility support for v1.3 ("legacy cmb"), but Keith suggested we just drop that and keep this device compliant.

What we could do is allow the spec version to be chosen with a parameter?

Uhm. Maybe not. I gave this some more thought.

Adding a device parameter to choose the specification version requires us to maintain QEMU "compat" properties across different QEMU version. I'm not sure we want that for something like this.

Maybe the best course of action actually *is* an 'x-legacy-cmb' parameter.

#31 lore raw

Re: [PATCH v2 10/12] hw/block/nvme: move cmb logic to v1.4

Quoted 26 lines · Klaus Jensen
Quoted 25 lines · Klaus Jensen
Quoted 15 lines · Minwoo Im
Quoted 14 lines · Klaus Jensen
Quoted 12 lines · Minwoo Im
Quoted 7 lines
Quoted 3 lines

Yes, CMB in v1.4 is not backward-compatible, but is it okay to move onto the CMB v1.4 from v1.3 without supporting the v1.3 usage on this device model?

Next patch moves to v1.4. I wanted to split it because the "bump" patch also adds a couple of other v1.4 requires fields. But I understand that this is slightly wrong.

Sorry, I meant I'd like to have CMB for v1.3 support along with the v1.4 support in this device model separately. Maybe I missed the linux-nvme mailing list for CMB v1.4, but is there no plan to keep supportin CMB v1.3 in NVMe driver?

Quoted 4 lines
Quoted 2 lines

I posted a patch on linux-nvme for v1.4 support in the kernel. It will support both the v1.3 and v1.4 behavior :)

Then, can we maintain CMB v1.3 also in QEMU also along with v1.4 ? :)

My first version of this patch actually included compatibility support for v1.3 ("legacy cmb"), but Keith suggested we just drop that and keep this device compliant.

Quoted 8 lines
Quoted 2 lines

What we could do is allow the spec version to be chosen with a parameter?

Uhm. Maybe not. I gave this some more thought.

Adding a device parameter to choose the specification version requires us to maintain QEMU "compat" properties across different QEMU version. I'm not sure we want that for something like this.

Agreed. The default should head for latest one.

Quoted 3 lines

Maybe the best course of action actually *is* an 'x-legacy-cmb' parameter.

This looks fine.

As kernel driver does not remove the v1.3 CMB support, then it would be great if QEMU suports that also.

#32 lore raw

PATCH v2 11/12 hw/block/nvme: bump to v1.4

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

With the new CMB logic in place, bump the implemented specification version to v1.4.

This requires adding the setting the CNTRLTYPE field and modifying the VWC field since 0x00 is no longer a valid value for bits 2:1.

---
 include/block/nvme.h | 3 ++-
 hw/block/nvme.c      | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)
include/block/nvme.h +2 -1
index 7dcd8f9b4e78..c34343c13a3c 100644
@@ -949,7 +949,8 @@ typedef struct QEMU_PACKED NvmeIdCtrl {
     uint32_t    rtd3e;
     uint32_t    oaes;
     uint32_t    ctratt;
-    uint8_t     rsvd100[12];
+    uint8_t     rsvd100[11];
+    uint8_t     cntrltype;
     uint8_t     fguid[16];
     uint8_t     rsvd128[128];
     uint16_t    oacs;
hw/block/nvme.c +3 -2
index a9b11a193c59..992665376a71 100644
@@ -106,7 +106,7 @@
 
 #define NVME_MAX_IOQPAIRS 0xffff
 #define NVME_DB_SIZE  4
-#define NVME_SPEC_VER 0x00010300
+#define NVME_SPEC_VER 0x00010400
 #define NVME_CMB_BIR 2
 #define NVME_PMR_BIR 4
 #define NVME_TEMPERATURE 0x143
@@ -4404,6 +4404,7 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
     id->mdts = n->params.mdts;
     id->ver = cpu_to_le32(NVME_SPEC_VER);
     id->oacs = cpu_to_le16(0);
+    id->cntrltype = 0x1;
 
     /*
      * Because the controller always completes the Abort command immediately,
@@ -4432,7 +4433,7 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
                            NVME_ONCS_FEATURES | NVME_ONCS_DSM |
                            NVME_ONCS_COMPARE);
 
-    id->vwc = 0x1;
+    id->vwc = (0x2 << 1) | 0x1;
     id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN |
                            NVME_CTRL_SGLS_BITBUCKET);
 
Signature · 1 line
2.30.0
#33 lore raw

PATCH v2 12/12 hw/block/nvme: lift cmb restrictions

In reply to Klaus Jensen

From: Klaus Jensen <[email protected]>

The controller now implements v1.4 and we can lift the restrictions on CMB Data Pointer and Command Independent Locations Support (CDPCILS) and CMB Data Pointer Mixed Locations Support (CDPMLS) since the device really does not care about mixed host/cmb pointers in those cases.

---
 hw/block/nvme.c | 33 ++-------------------------------
 1 file changed, 2 insertions(+), 31 deletions(-)
hw/block/nvme.c +2 -31
index 992665376a71..23a836a343e2 100644
@@ -509,7 +509,6 @@ static uint16_t nvme_map_prp(NvmeCtrl *n, uint64_t prp1, uint64_t prp2,
     trans_len = MIN(len, trans_len);
     int num_prps = (len >> n->page_bits) + 1;
     uint16_t status;
-    bool prp_list_in_cmb = false;
     int ret;
 
     QEMUSGList *qsg = &req->qsg;
@@ -535,10 +534,6 @@ static uint16_t nvme_map_prp(NvmeCtrl *n, uint64_t prp1, uint64_t prp2,
             uint32_t nents, prp_trans;
             int i = 0;
 
-            if (nvme_addr_is_cmb(n, prp2)) {
-                prp_list_in_cmb = true;
-            }
-
             nents = (len + n->page_size - 1) >> n->page_bits;
             prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
             ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
@@ -555,10 +550,6 @@ static uint16_t nvme_map_prp(NvmeCtrl *n, uint64_t prp1, uint64_t prp2,
                         return NVME_INVALID_PRP_OFFSET | NVME_DNR;
                     }
 
-                    if (prp_list_in_cmb != nvme_addr_is_cmb(n, prp_ent)) {
-                        return NVME_INVALID_USE_OF_CMB | NVME_DNR;
-                    }
-
                     i = 0;
                     nents = (len + n->page_size - 1) >> n->page_bits;
                     prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
@@ -692,7 +683,6 @@ static uint16_t nvme_map_sgl(NvmeCtrl *n, QEMUSGList *qsg, QEMUIOVector *iov,
     uint64_t nsgld;
     uint32_t seg_len;
     uint16_t status;
-    bool sgl_in_cmb = false;
     hwaddr addr;
     int ret;
 
@@ -714,18 +704,6 @@ static uint16_t nvme_map_sgl(NvmeCtrl *n, QEMUSGList *qsg, QEMUIOVector *iov,
         goto out;
     }
 
-    /*
-     * If the segment is located in the CMB, the submission queue of the
-     * request must also reside there.
-     */
-    if (nvme_addr_is_cmb(n, addr)) {
-        if (!nvme_addr_is_cmb(n, req->sq->dma_addr)) {
-            return NVME_INVALID_USE_OF_CMB | NVME_DNR;
-        }
-
-        sgl_in_cmb = true;
-    }
-
     for (;;) {
         switch (NVME_SGL_TYPE(sgld->type)) {
         case NVME_SGL_DESCR_TYPE_SEGMENT:
@@ -814,15 +792,6 @@ static uint16_t nvme_map_sgl(NvmeCtrl *n, QEMUSGList *qsg, QEMUIOVector *iov,
         if (status) {
             goto unmap;
         }
-
-        /*
-         * If the next segment is in the CMB, make sure that the sgl was
-         * already located there.
-         */
-        if (sgl_in_cmb != nvme_addr_is_cmb(n, addr)) {
-            status = NVME_INVALID_USE_OF_CMB | NVME_DNR;
-            goto unmap;
-        }
     }
 
 out:
@@ -3739,6 +3708,8 @@ static int nvme_start_ctrl(NvmeCtrl *n)
 
 static void nvme_cmb_enable_regs(NvmeCtrl *n)
 {
+    NVME_CMBLOC_SET_CDPCILS(n->bar.cmbloc, 1);
+    NVME_CMBLOC_SET_CDPMLS(n->bar.cmbloc, 1);
     NVME_CMBLOC_SET_BIR(n->bar.cmbloc, NVME_CMB_BIR);
 
     NVME_CMBSZ_SET_SQS(n->bar.cmbsz, 1);
Signature · 1 line
2.30.0