nvme: allow cmb and pmr emulation on same device
PATCH 1/1 nvme: allow cmb and pmr emulation on same device
This patch series does following:
- Fixes problem where CMBS bit was not set in controller capabilities register, so support for CMB was not correctly advertised to guest. This is resend of patch that has been submitted and reviewed by Klaus [1]
- Introduces BAR4 sharing between MSI-X vectors and CMB. This allows to have CMB and PMR emulated on the same device. This extension was indicated by Keith [2]
[1]: https://lore.kernel.org/qemu-devel/[email protected]/
[2]: https://lore.kernel.org/qemu-devel/[email protected]/
PATCH 1/2 nvme: indicate CMB support through controller capabilities register
This patch sets CMBS bit in controller capabilities register when user configures NVMe driver with CMB support, so capabilites are correctly reported to guest OS.
--- hw/block/nvme.c | 2 ++ include/block/nvme.h | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-)
hw/block/nvme.c +2 -0
@@ -1449,6 +1449,8 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) n->bar.intmc = n->bar.intms =0 ;if (n->cmb_size_mb) { +/* Contoller capabilities */ +NVME_CAP_SET_CMBS (n->bar.cap,1 );NVME_CMBLOC_SET_BIR (n->bar.cmbloc,2 );NVME_CMBLOC_SET_OFST (n->bar.cmbloc,0 );
include/block/nvme.h +5 -1
@@ -35,6 +35,7 @@ enum NvmeCapShift {CAP_MPSMIN_SHIFT =48 ,CAP_MPSMAX_SHIFT =52 ,CAP_PMR_SHIFT =56 , + CAP_CMB_SHIFT = 57, }; enum NvmeCapMask {
@@ -48,6 +49,7 @@ enum NvmeCapMask {CAP_MPSMIN_MASK =0xf ,CAP_MPSMAX_MASK =0xf ,CAP_PMR_MASK =0x1 , + CAP_CMB_MASK = 0x1, };#define NVME_CAP_MQES (cap) (((cap) >> CAP_MQES_SHIFT) & CAP_MQES_MASK)
@@ -78,8 +80,10 @@ enum NvmeCapMask { << CAP_MPSMIN_SHIFT)#define NVME_CAP_SET_MPSMAX (cap, val) (cap |= (uint64_t)(val & CAP_MPSMAX_MASK)\ << CAP_MPSMAX_SHIFT) -#define NVME_CAP_SET_PMRS (cap, val) (cap |= (uint64_t)(val & CAP_PMR_MASK)\ +#define NVME_CAP_SET_PMRS (cap, val) (cap |= (uint64_t)(val & CAP_PMR_MASK) \ << CAP_PMR_SHIFT) +#define NVME_CAP_SET_CMBS (cap, val) (cap |= (uint64_t)(val & CAP_CMB_MASK) \ + << CAP_CMB_SHIFT)enum NvmeCcShift {CC_EN_SHIFT =0 ,
Signature · 1 line
2.21.1
PATCH 2/2 nvme: allow cmb and pmr to be enabled on same device
In reply to Andrzej JakowskiSo far it was not possible to have CMB and PMR emulated on the same device, because BAR2 was used exclusively either of PMR or CMB. This patch places CMB at BAR4 offset so it not conflicts with MSI-X vectors.
--- hw/block/nvme.c | 127 +++++++++++++++++++++++++++++-------------- hw/block/nvme.h | 3 +- include/block/nvme.h | 4 +- 3 files changed, 91 insertions(+), 43 deletions(-)
hw/block/nvme.c +86 -41
@@ -22,12 +22,12 @@* [pmrdev=<mem_backend_file_id>,] \ * num_queues=<N[optional]> * -* Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at -* offset 0 in BAR2 and supports only WDS, RDS and SQS for now. +* Note cmb_size_mb denotes size of CMB in MB. CMB when configured is assumed +* to be resident in BAR4 at certain offset - this is because BAR4 is also +* used for storing MSI-X table that is available at offset 0 in BAR4. * -* cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation -* in available BAR's. cmb_size_mb= will take precedence over pmrdev= when -* both provided. +* pmrdev is assumed to be resident in BAR2. When configured it consumes whole +* BAR2 exclusively. * Enabling pmr emulation can be achieved by pointing to memory-backend-file. * For example: * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
@@ -64,9 +64,10 @@ static void nvme_process_sq(void *opaque);static void nvme_addr_read (NvmeCtrl *n,hwaddr addr,void *buf,int size) { -if (n->cmbsz && addr >= n->ctrl_mem.addr && - addr < (n->ctrl_mem.addr +int128_get64 (n->ctrl_mem.size))) { -memcpy (buf, (void *)&n->cmbuf[addr - n->ctrl_mem.addr], size); +hwaddr cmb_addr = n->bar4.addr + n->cmb_offset; +if (n->cmbsz && addr >= cmb_addr && + (addr + size) <= (cmb_addr +NVME_CMBSZ_GETSIZE (n->bar.cmbsz))) { +memcpy (buf, (void *)&n->cmbuf[addr - cmb_addr], size); }else {pci_dma_read (&n->parent_obj, addr, buf, size); }
@@ -152,17 +153,18 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1, uint64_t prp2, uint32_t len, NvmeCtrl *n) { hwaddr trans_len = n->page_size - (prp1 % n->page_size); +hwaddr cmb_addr = n->bar4.addr + n->cmb_offset; trans_len =MIN (len, trans_len);int num_prps = (len >> n->page_bits) +1 ;if (unlikely (!prp1)) {trace_nvme_err_invalid_prp ();return NVME_INVALID_FIELD |NVME_DNR ; - }else if (n->cmbsz && prp1 >= n->ctrl_mem.addr && - prp1 < n->ctrl_mem.addr +int128_get64 (n->ctrl_mem.size)) { + }else if (n->cmbsz && prp1 >= cmb_addr && + prp1 < cmb_addr +int128_get64 (n->bar4.size)) { qsg->nsg =0 ;qemu_iovec_init (iov, num_prps); -qemu_iovec_add (iov, (void *)&n->cmbuf[prp1 - n->ctrl_mem.addr], trans_len); +qemu_iovec_add (iov, (void *)&n->cmbuf[prp1 - cmb_addr], trans_len); }else {pci_dma_sglist_init (qsg, &n->parent_obj, num_prps);qemu_sglist_add (qsg, prp1, trans_len);
@@ -207,7 +209,8 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,if (qsg->nsg){qemu_sglist_add (qsg, prp_ent, trans_len); }else { -qemu_iovec_add (iov, (void *)&n->cmbuf[prp_ent - n->ctrl_mem.addr], trans_len); +qemu_iovec_add (iov, (void *)&n->cmbuf[prp_ent - cmb_addr], + trans_len); } len -= trans_len; i++;
@@ -220,7 +223,8 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,if (qsg->nsg) {qemu_sglist_add (qsg, prp2, len); }else { -qemu_iovec_add (iov, (void *)&n->cmbuf[prp2 - n->ctrl_mem.addr], trans_len); +qemu_iovec_add (iov, (void *)&n->cmbuf[prp2 - cmb_addr], + trans_len); } } }
@@ -1342,6 +1346,71 @@ static const MemoryRegionOps nvme_cmb_ops = { }, }; +#define NVME_MSIX_BIR (4) +static void nvme_bar4_init (PCIDevice *pci_dev) +{ +NvmeCtrl *n =NVME (pci_dev); +int status; +uint64_t bar_size =4096 ; +uint32_t nvme_pba_offset = bar_size /2 ; +uint32_t nvme_pba_size =QEMU_ALIGN_UP (n->num_queues,64 ) /8 ; +uint32_t cmb_size_units; + +if (n->num_queues *PCI_MSIX_ENTRY_SIZE > nvme_pba_offset) { + nvme_pba_offset = n->num_queues *PCI_MSIX_ENTRY_SIZE ; + } + +if (nvme_pba_offset + nvme_pba_size >4096 ) { + bar_size = nvme_pba_offset + nvme_pba_size; + } + +if (n->cmb_size_mb) { +/* Contoller capabilities */ +NVME_CAP_SET_CMBS (n->bar.cap,1 ); + +NVME_CMBSZ_SET_SQS (n->bar.cmbsz,1 ); +NVME_CMBSZ_SET_CQS (n->bar.cmbsz,0 ); +NVME_CMBSZ_SET_LISTS (n->bar.cmbsz,0 ); +NVME_CMBSZ_SET_RDS (n->bar.cmbsz,1 ); +NVME_CMBSZ_SET_WDS (n->bar.cmbsz,1 ); +NVME_CMBSZ_SET_SZU (n->bar.cmbsz,2 );/* MBs */ +NVME_CMBSZ_SET_SZ (n->bar.cmbsz, n->cmb_size_mb); + + cmb_size_units =NVME_CMBSZ_GETSIZEUNITS (n->bar.cmbsz); + n->cmb_offset =QEMU_ALIGN_UP (bar_size, cmb_size_units); + +NVME_CMBLOC_SET_BIR (n->bar.cmbloc,NVME_MSIX_BIR ); +NVME_CMBLOC_SET_OFST (n->bar.cmbloc, n->cmb_offset / cmb_size_units); + + n->cmbloc = n->bar.cmbloc; + n->cmbsz = n->bar.cmbsz; + + n->cmbuf =g_malloc0 (NVME_CMBSZ_GETSIZE (n->bar.cmbsz)); + + bar_size += n->cmb_offset; + bar_size +=NVME_CMBSZ_GETSIZE (n->bar.cmbsz); + } + + bar_size =pow2ceil (bar_size); + +memory_region_init_io (&n->bar4,OBJECT (n), &nvme_cmb_ops, n, +"nvme-bar4" , bar_size); + + status =msix_init (pci_dev, n->num_queues, + &n->bar4,NVME_MSIX_BIR ,0 , + &n->bar4,NVME_MSIX_BIR , nvme_pba_offset, +0 ,NULL ); + +if (status) { +return ; + } + +pci_register_bar (pci_dev,NVME_MSIX_BIR , +PCI_BASE_ADDRESS_SPACE_MEMORY |PCI_BASE_ADDRESS_MEM_TYPE_64 | +PCI_BASE_ADDRESS_MEM_PREFETCH , &n->bar4); + +} +static void nvme_realize (PCIDevice *pci_dev,Error **errp) {NvmeCtrl *n =NVME (pci_dev);
@@ -1372,7 +1441,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)return ; } -if (!n->cmb_size_mb && n->pmrdev) { +if (n->pmrdev) {if (host_memory_backend_is_mapped (n->pmrdev)) {char *path =object_get_canonical_path_component (OBJECT (n->pmrdev));error_setg (errp,"can't use already busy memdev: %s" , path);
@@ -1413,7 +1482,6 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)pci_register_bar (pci_dev,0 ,PCI_BASE_ADDRESS_SPACE_MEMORY |PCI_BASE_ADDRESS_MEM_TYPE_64 , &n->iomem); -msix_init_exclusive_bar (pci_dev, n->num_queues,4 ,NULL ); id->vid =cpu_to_le16 (pci_get_word (pci_conf +PCI_VENDOR_ID )); id->ssvid =cpu_to_le16 (pci_get_word (pci_conf +PCI_SUBSYSTEM_VENDOR_ID ));
@@ -1445,35 +1513,12 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)NVME_CAP_SET_CSS (n->bar.cap,1 );NVME_CAP_SET_MPSMAX (n->bar.cap,4 ); +nvme_bar4_init (pci_dev); + n->bar.vs =0x00010200 ; n->bar.intmc = n->bar.intms =0 ; -if (n->cmb_size_mb) { -/* Contoller capabilities */ -NVME_CAP_SET_CMBS (n->bar.cap,1 ); - -NVME_CMBLOC_SET_BIR (n->bar.cmbloc,2 ); -NVME_CMBLOC_SET_OFST (n->bar.cmbloc,0 ); - -NVME_CMBSZ_SET_SQS (n->bar.cmbsz,1 ); -NVME_CMBSZ_SET_CQS (n->bar.cmbsz,0 ); -NVME_CMBSZ_SET_LISTS (n->bar.cmbsz,0 ); -NVME_CMBSZ_SET_RDS (n->bar.cmbsz,1 ); -NVME_CMBSZ_SET_WDS (n->bar.cmbsz,1 ); -NVME_CMBSZ_SET_SZU (n->bar.cmbsz,2 );/* MBs */ -NVME_CMBSZ_SET_SZ (n->bar.cmbsz, n->cmb_size_mb); - - n->cmbloc = n->bar.cmbloc; - n->cmbsz = n->bar.cmbsz; - - n->cmbuf =g_malloc0 (NVME_CMBSZ_GETSIZE (n->bar.cmbsz)); -memory_region_init_io (&n->ctrl_mem,OBJECT (n), &nvme_cmb_ops, n, -"nvme-cmb" ,NVME_CMBSZ_GETSIZE (n->bar.cmbsz)); -pci_register_bar (pci_dev,NVME_CMBLOC_BIR (n->bar.cmbloc), -PCI_BASE_ADDRESS_SPACE_MEMORY |PCI_BASE_ADDRESS_MEM_TYPE_64 | -PCI_BASE_ADDRESS_MEM_PREFETCH , &n->ctrl_mem); - - }else if (n->pmrdev) { +if (n->pmrdev) {/* Controller Capabilities register */ NVME_CAP_SET_PMRS (n->bar.cap,1 );
hw/block/nvme.h +2 -1
@@ -60,7 +60,7 @@ typedef struct NvmeNamespace { typedef struct NvmeCtrl { PCIDevice parent_obj; MemoryRegion iomem; - MemoryRegion ctrl_mem; + MemoryRegion bar4; NvmeBar bar; BlockConf conf;
@@ -77,6 +77,7 @@ typedef struct NvmeCtrl {uint32_t cmb_size_mb;uint32_t cmbsz;uint32_t cmbloc; +uint32_t cmb_offset;uint8_t *cmbuf;uint64_t irq_status;uint64_t host_timestamp;/* Timestamp sent by the host */
include/block/nvme.h +3 -1
@@ -216,9 +216,11 @@ enum NvmeCmbszMask { (cmbsz |= (uint64_t)(val & CMBSZ_SZU_MASK) << CMBSZ_SZU_SHIFT)#define NVME_CMBSZ_SET_SZ (cmbsz, val) \ (cmbsz |= (uint64_t)(val & CMBSZ_SZ_MASK) << CMBSZ_SZ_SHIFT) +#define NVME_CMBSZ_GETSIZEUNITS (cmbsz) \ + (1 << (12 + 4 * NVME_CMBSZ_SZU(cmbsz)))#define NVME_CMBSZ_GETSIZE (cmbsz) \ - (NVME_CMBSZ_SZ(cmbsz) * (1 << (12 + 4 * NVME_CMBSZ_SZU(cmbsz)))) + (NVME_CMBSZ_SZ(cmbsz) * NVME_CMBSZ_GETSIZEUNITS(cmbsz))enum NvmePmrcapShift {PMRCAP_RDS_SHIFT =3 ,
Signature · 1 line
2.21.1
Re: [PATCH v1 2/2] nvme: allow cmb and pmr to be enabled on same device
Quoted 4 lines · Andrzej Jakowski
So far it was not possible to have CMB and PMR emulated on the same device, because BAR2 was used exclusively either of PMR or CMB. This patch places CMB at BAR4 offset so it not conflicts with MSI-X vectors.
Hi Andrzej,
Thanks for doing this, it's a nice addition!
Though, I would prefer that the table and pba was located in BAR0 and keeping BAR4 for exclusive CMB use. I'm no expert on this, but is it ok to have the table and pba in prefetchable memory? Having it "together" with the other controller-level configuration memory just feels more natural to me, but I'm not gonna put my foot down.
Using BAR0 would also slightly simplify the patch since no changes would be required for the CMB path.
Also, can you rebase this on Kevin's block branch? There are a bunch of refactoring patches that changes the realization code, so this patch doesn't apply at all.
Quoted 16 lines
Signed-off-by Andrzej Jakowski <[email protected]>--- hw/block/nvme.c | 127 +++++++++++++++++++++++++++++-------------- hw/block/nvme.h | 3 +- include/block/nvme.h | 4 +- 3 files changed, 91 insertions(+), 43 deletions(-)hw/block/nvme.c +0
-1@@ -22,12 +22,12 @@* [pmrdev=<mem_backend_file_id>,] \ * num_queues=<N[optional]> * -* Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
hw/block/nvme.c line 30
Quoted 10 lines
hw/block/nvme.c +5
-4-* offset 0 in BAR2 and supports only WDS, RDS and SQS for now. +* Note cmb_size_mb denotes size of CMB in MB. CMB when configured is assumed +* to be resident in BAR4 at certain offset - this is because BAR4 is also +* used for storing MSI-X table that is available at offset 0 in BAR4. * -* cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation -* in available BAR's. cmb_size_mb= will take precedence over pmrdev= when -* both provided. +* pmrdev is assumed to be resident in BAR2. When configured it consumes whole +* BAR2 exclusively.
Actually it uses both BAR2 and BAR3 since its 64 bits.
Quoted 12 lines
hw/block/nvme.c +8
-0@@ -1342,6 +1346,71 @@ static const MemoryRegionOps nvme_cmb_ops = { }, }; +#define NVME_MSIX_BIR (4) +static void nvme_bar4_init (PCIDevice *pci_dev) +{ +NvmeCtrl *n =NVME (pci_dev); +int status; +uint64_t bar_size =4096 ; +uint32_t nvme_pba_offset = bar_size /2 ; +uint32_t nvme_pba_size =QEMU_ALIGN_UP (n->num_queues,64 ) /8 ;
hw/block/nvme.c static const MemoryRegionOps nvme_cmb_ops = { · line 1366
Quoted 10 lines
hw/block/nvme.c +10
-0+uint32_t cmb_size_units; + +if (n->num_queues *PCI_MSIX_ENTRY_SIZE > nvme_pba_offset) { + nvme_pba_offset = n->num_queues *PCI_MSIX_ENTRY_SIZE ; + } + +if (nvme_pba_offset + nvme_pba_size >4096 ) { + bar_size = nvme_pba_offset + nvme_pba_size; + } +
This is migration compatibility stuff that is not needed because the nvme device is unmigratable anyway.
Re: [PATCH v1 2/2] nvme: allow cmb and pmr to be enabled on same device
Quoted 8 lines · Klaus Jensen
Quoted 4 lines · Andrzej Jakowski
So far it was not possible to have CMB and PMR emulated on the same device, because BAR2 was used exclusively either of PMR or CMB. This patch places CMB at BAR4 offset so it not conflicts with MSI-X vectors.
Hi Andrzej,
Quoted 7 lines
Thanks for doing this, it's a nice addition!
Though, I would prefer that the table and pba was located in BAR0 and keeping BAR4 for exclusive CMB use. I'm no expert on this, but is it ok to have the table and pba in prefetchable memory? Having it "together" with the other controller-level configuration memory just feels more natural to me, but I'm not gonna put my foot down.
Hi Klaus,
Thx for your feedback!
I don't think it matters if MSIX table is in prefetchable vs non-prefetchable memory.
My understanding is that spec allows MSIX and PBA to be in any BAR and offset. I understand your preference and at the same time think that since it is not in violation of the spec why don't we leave it as-is? Does anybody know what's typical approach for real devices?
Quoted 7 lines
Using BAR0 would also slightly simplify the patch since no changes would be required for the CMB path.
Also, can you rebase this on Kevin's block branch? There are a bunch of refactoring patches that changes the realization code, so this patch doesn't apply at all.
Yep will reabse it.
Quoted 21 lines
Quoted 21 lines
Signed-off-by Andrzej Jakowski <[email protected]>--- hw/block/nvme.c | 127 +++++++++++++++++++++++++++++-------------- hw/block/nvme.h | 3 +- include/block/nvme.h | 4 +- 3 files changed, 91 insertions(+), 43 deletions(-)hw/block/nvme.c +3
-2@@ -22,12 +22,12 @@* [pmrdev=<mem_backend_file_id>,] \ * num_queues=<N[optional]> * -* Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at -* offset 0 in BAR2 and supports only WDS, RDS and SQS for now. +* Note cmb_size_mb denotes size of CMB in MB. CMB when configured is assumed +* to be resident in BAR4 at certain offset - this is because BAR4 is also +* used for storing MSI-X table that is available at offset 0 in BAR4.
Quoted 8 lines
hw/block/nvme.c line 30
Quoted 6 lines
hw/block/nvme.c +2
-3* -* cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation -* in available BAR's. cmb_size_mb= will take precedence over pmrdev= when -* both provided. +* pmrdev is assumed to be resident in BAR2. When configured it consumes whole +* BAR2 exclusively. Actually it uses both BAR2 and BAR3 since its 64 bits.
Correct. That's what I implied here w/o actual verbiage. I can extend it to add that information.
Quoted 18 lines
Quoted 18 lines
hw/block/nvme.c +13
-0@@ -1342,6 +1346,71 @@ static const MemoryRegionOps nvme_cmb_ops = { }, }; +#define NVME_MSIX_BIR (4) +static void nvme_bar4_init (PCIDevice *pci_dev) +{ +NvmeCtrl *n =NVME (pci_dev); +int status; +uint64_t bar_size =4096 ; +uint32_t nvme_pba_offset = bar_size /2 ; +uint32_t nvme_pba_size =QEMU_ALIGN_UP (n->num_queues,64 ) /8 ; +uint32_t cmb_size_units; + +if (n->num_queues *PCI_MSIX_ENTRY_SIZE > nvme_pba_offset) { + nvme_pba_offset = n->num_queues *PCI_MSIX_ENTRY_SIZE ; + }
Quoted 8 lines
hw/block/nvme.c static const MemoryRegionOps nvme_cmb_ops = { · line 1366
Quoted 5 lines
hw/block/nvme.c +5
-0+ +if (nvme_pba_offset + nvme_pba_size >4096 ) { + bar_size = nvme_pba_offset + nvme_pba_size; + } +This is migration compatibility stuff that is not needed because the nvme device is unmigratable anyway.
I don't understand that comment. Could you please explain more?
Re: [PATCH v1 2/2] nvme: allow cmb and pmr to be enabled on same device
Quoted 18 lines · Andrzej Jakowski
Quoted 8 lines · Klaus Jensen
Quoted 4 lines · Andrzej Jakowski
So far it was not possible to have CMB and PMR emulated on the same device, because BAR2 was used exclusively either of PMR or CMB. This patch places CMB at BAR4 offset so it not conflicts with MSI-X vectors.
Hi Andrzej,
Quoted 7 lines
Thanks for doing this, it's a nice addition!
Though, I would prefer that the table and pba was located in BAR0 and keeping BAR4 for exclusive CMB use. I'm no expert on this, but is it ok to have the table and pba in prefetchable memory? Having it "together" with the other controller-level configuration memory just feels more natural to me, but I'm not gonna put my foot down.
Hi Klaus,
Quoted 7 lines
Thx for your feedback!
I don't think it matters if MSIX table is in prefetchable vs non-prefetchable memory.
My understanding is that spec allows MSIX and PBA to be in any BAR and offset. I understand your preference and at the same time think that since it is not in violation of the spec why don't we leave it as-is? Does anybody know what's typical approach for real devices?
On the SSD in my system, lspci -vv shows:
Capabilities: [b0] MSI-X: Enable+ Count=33 Masked-
Vector table: BAR=0 offset=00002000
PBA: BAR=4 offset=00000000So, the table is in BAR0 but the PBA is in BAR4. Oh well.
Quoted 25 lines
Quoted 17 lines
Quoted 17 lines
hw/block/nvme.c +13
-0@@ -1342,6 +1346,71 @@ static const MemoryRegionOps nvme_cmb_ops = { }, }; +#define NVME_MSIX_BIR (4) +static void nvme_bar4_init (PCIDevice *pci_dev) +{ +NvmeCtrl *n =NVME (pci_dev); +int status; +uint64_t bar_size =4096 ; +uint32_t nvme_pba_offset = bar_size /2 ; +uint32_t nvme_pba_size =QEMU_ALIGN_UP (n->num_queues,64 ) /8 ; +uint32_t cmb_size_units; + +if (n->num_queues *PCI_MSIX_ENTRY_SIZE > nvme_pba_offset) { + nvme_pba_offset = n->num_queues *PCI_MSIX_ENTRY_SIZE ; + }Quoted 8 lines
hw/block/nvme.c static const MemoryRegionOps nvme_cmb_ops = { · line 1366
Quoted 5 lines
hw/block/nvme.c +5
-0+ +if (nvme_pba_offset + nvme_pba_size >4096 ) { + bar_size = nvme_pba_offset + nvme_pba_size; + } +This is migration compatibility stuff that is not needed because the nvme device is unmigratable anyway.
Quoted 3 lines
I don't understand that comment. Could you please explain more?
It looks like you cribbed this code from msix_init_exclusive_bar(). All that fuzz about the PBA starting in the upper half (nvme_pba_offset = bar_size / 2) for nentries lower or equal to 128 is for migration compatibility (migrating a VM from an old version of QEMU to a newer one). It is something that was added when the location of the msix table and pba became dynamic (it used to be static). But, since the nvme device is marked as unmigratable (VMStateDescription.unmigratable = 1), I believe these special cases are irrelevant.