I/O performance with block sizes > 128k
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
filename=/dev/nvme0n1
rw=randread
direct=1
time_based=1
randrepeat=1
gtod_reduce=1
fio --readonly --ioengine=io_uring --iodepth 1024 --fixedbufs --hipri --numjob
The problem seems to be related to switching from prp_small_pool to prp_page_pool; the former is optimized for I/O between 4k and 128k.
Expanding the small pool size to cover up to 256k increases the performance. I'm not sure however if this is the proper and general fix. For one thing, expanding the pool size bumps the numbers for a short burst test (10 sec) but the numbers drop again significantly during a longer test. The behavior is not unique to io_uring either. Included below are a couple of 256k fio pvsync2 tests as well.
A related question, is it required for dma pool allocations to use GFP_ATOMIC? Looks they can only be called from nvme_queue_rq().Just as a test, I changed the flag to GFP_NOWAIT, and that seems to be ok.
drivers/nvme/host/pci.c +0 -0
@@ -40,6 +40,8 @@
#define NVME_MAX_KB_SZ 4096 #define NVME_MAX_SEGS 127
+#define PRP_SMALL_SZ 512
+
static int use_threaded_interrupts; module_param(use_threaded_interrupts, int, 0);
@@ -618,7 +620,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev }
nprps = DIV_ROUND_UP(length, page_size);
- if (nprps <= (256 / 8)) {
+ if (nprps <= (PRP_SMALL_SZ / 8)) {
pool = dev->prp_small_pool;
iod->npages = 0;
} else {
@@ -626,7 +628,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev iod->npages = 1;
}
- prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
+ prp_list = dma_pool_alloc(pool, GFP_NOWAIT, &prp_dma);
if (!prp_list) {
iod->first_dma = dma_addr;
iod->npages = -1;
@@ -638,7 +640,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev for (;;) {
if (i == page_size >> 3) {
__le64 *old_prp_list = prp_list;
- prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); + prp_list = dma_pool_alloc(pool, GFP_NOWAIT, &prp_dma); if (!prp_list)
return BLK_STS_RESOURCE; list[iod->npages++] = prp_list;
@@ -713,7 +715,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev return BLK_STS_OK;
}
- if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
+ if (entries <= (PRP_SMALL_SZ / sizeof(struct nvme_sgl_desc))) { pool = dev->prp_small_pool;
iod->npages = 0;
} else {
@@ -721,7 +723,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev iod->npages = 1;
}
- sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
+ sg_list = dma_pool_alloc(pool, GFP_NOWAIT, &sgl_dma);
if (!sg_list) {
iod->npages = -1;
return BLK_STS_RESOURCE;
@@ -737,7 +739,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev struct nvme_sgl_desc *old_sg_desc = sg_list; struct nvme_sgl_desc *link = &old_sg_desc[i - 1];
- sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma); + sg_list = dma_pool_alloc(pool, GFP_NOWAIT, &sgl_dma); if (!sg_list)
return BLK_STS_RESOURCE;
@@ -814,7 +816,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, stru }
iod->dma_len = 0;
- iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC); + iod->sg = mempool_alloc(dev->iod_mempool, GFP_NOWAIT); if (!iod->sg)
return BLK_STS_RESOURCE;
sg_init_table(iod->sg, blk_rq_nr_phys_segments(req)); @@ -2475,9 +2477,9 @@ static int nvme_setup_prp_pools(struct nvme_dev *dev) if (!dev->prp_page_pool)
return -ENOMEM;
- /* Optimisation for I/Os between 4k and 128k */
- dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev, - 256, 256, 0); + /* Optimisation for I/Os between 4k and 256k */
+ dev->prp_small_pool = dma_pool_create("prp list small", dev->dev,
+ PRP_SMALL_SZ, PRP_SMALL_SZ, 0);
if (!dev->prp_small_pool) { dma_pool_destroy(dev->prp_page_pool);
return -ENOMEM;
prp_pool 256
bw=3338MiB/s (3500MB/s) 128k, 10sec bw=868MiB/s (911MB/s) 144k, 10sec bw=1607MiB/s (1685MB/s) 256k, 10sec
prp_pool 512
bw=3345MiB/s (3507MB/s) 128k, 10sec bw=3453MiB/s (3621MB/s) 144k, 10sec bw=4603MiB/s (4826MB/s) 256k, 10sec bw=2982MiB/s (3127MB/s) 128k, 5min bw=1762MiB/s (1847MB/s) 256k, 5min bw=3579MiB/s (3753MB/s) 256k, 10sec, pvsync2 bw=1752MiB/s (1837MB/s) 256k, 5min, pvsync2
prp_pool 512 , GFP_NOWAIT bw=3295MiB/s (3455MB/s) 128k, 10sec bw=3467MiB/s (3635MB/s) 144k, 10sec bw=4486MiB/s (4704MB/s) 256k, 10sec bw=3013MiB/s (3159MB/s) 128k, 5min bw=1764MiB/s (1849MB/s) 256k, 5min
--bijan
_______________________________________________
linux-nvme mailing list [email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Keith Busch
maintainerQuoted 16 lines · Bijan Mottahedeh
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
filename=/dev/nvme0n1
rw=randread
direct=1
time_based=1
randrepeat=1
gtod_reduce=1fio --readonly --ioengine=io_uring --iodepth 1024 --fixedbufs --hipri --numjob
The problem seems to be related to switching from
prp_small_pooltoprp_page_pool; the former is optimized for I/O between 4k and 128k.
Quoted 6 lines
Expanding the small pool size to cover up to 256k increases the performance. I'm not sure however if this is the proper and general fix. For one thing, expanding the pool size bumps the numbers for a short burst test (10 sec) but the numbers drop again significantly during a longer test. The behavior is not unique to
io_uringeither. Included below are a couple of 256k fio pvsync2 tests as well.
I am surprised you're seeing such a drop just from the prp pool used.
What CPU architecture are you using? Reason I ask: the driver allocates PAGE_SIZE for the large prp pool, but we really want ctrl->page_size (always 4k). If your CPU architecture has a PAGE_SIZE larger than 4k, could you try the following?
---
drivers/nvme/host/pci.c +1 -1
@@ -2458,7 +2458,7 @@ static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)static int nvme_setup_prp_pools (struct nvme_dev *dev) { dev->prp_page_pool =dma_pool_create ("prp list page" , dev->dev, -PAGE_SIZE ,PAGE_SIZE ,0 ); +4096 ,4096 ,0 );if (!dev->prp_page_pool)return -ENOMEM ;
--
Quoted 3 lines
A related question, is it required for dma pool allocations to use GFP_ATOMIC? Looks they can only be called from nvme_queue_rq().Just as a test, I changed the flag to
GFP_NOWAIT, and that seems to be ok.
Yes, the atomic alloc is left over from a time when this happened under a spinlock. NOWAIT should be fine here.
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 23 lines · Keith Busch
Quoted 16 lines · Bijan Mottahedeh
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
filename=/dev/nvme0n1
rw=randread
direct=1
time_based=1
randrepeat=1
gtod_reduce=1fio --readonly --ioengine=io_uring --iodepth 1024 --fixedbufs --hipri --numjob
The problem seems to be related to switching from
prp_small_pooltoprp_page_pool; the former is optimized for I/O between 4k and 128k.Quoted 6 lines
Expanding the small pool size to cover up to 256k increases the performance. I'm not sure however if this is the proper and general fix. For one thing, expanding the pool size bumps the numbers for a short burst test (10 sec) but the numbers drop again significantly during a longer test. The behavior is not unique to
io_uringeither. Included below are a couple of 256k fio pvsync2 tests as well.
Quoted 6 lines
I am surprised you're seeing such a drop just from the prp pool used.
What CPU architecture are you using? Reason I ask: the driver allocates
PAGE_SIZEfor the large prp pool, but we really wantctrl->page_size(always 4k). If your CPU architecture has aPAGE_SIZElarger than 4k, could you try the following?
It's an x86_64 vm with 8GB of memory. Is the 4k pool size meant to support up to a 2MB i/o size then?
The main seems to be a lock contention, this what I see with the 256k test after running a while. However, I can't pinpoint the lock with perf lock; is there a better way to do that?
65.08% [kernel] [k] __pv_queued_spin_lock_slowpath 2.43% [kernel] [k] mutex_spin_on_owner
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 16
On-line CPU(s) list: 0-15
Thread(s) per core: 1
Core(s) per socket: 1
Socket(s): 16
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 63
Model name: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz Stepping: 2
CPU MHz: 2294.876
BogoMIPS: 4589.75
Virtualization: VT-x
Hypervisor vendor: KVM
Virtualization type: full
L1d cache: 32K
L1i cache: 32K
L2 cache: 4096K
L3 cache: 16384K
NUMA node0 CPU(s): 0-15
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cr
Quoted 17 lines
---
drivers/nvme/host/pci.c +1
-2@@ -2458,7 +2458,7 @@ static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)static int nvme_setup_prp_pools (struct nvme_dev *dev) { dev->prp_page_pool =dma_pool_create ("prp list page" , dev->dev, -PAGE_SIZE ,PAGE_SIZE ,0 ); +4096 ,4096 ,0 );if (!dev->prp_page_pool)return -ENOMEM ; --
Quoted 5 lines
Quoted 3 lines
A related question, is it required for dma pool allocations to use GFP_ATOMIC? Looks they can only be called from nvme_queue_rq().Just as a test, I changed the flag to
GFP_NOWAIT, and that seems to be ok.Yes, the atomic alloc is left over from a time when this happened under a spinlock. NOWAIT should be fine here.
_______________________________________________
linux-nvme mailing list [email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 35 lines · Bijan Mottahedeh
Quoted 23 lines · Keith Busch
Quoted 16 lines · Bijan Mottahedeh
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
filename=/dev/nvme0n1
rw=randread
direct=1
time_based=1
randrepeat=1
gtod_reduce=1fio --readonly --ioengine=io_uring --iodepth 1024 --fixedbufs --hipri --numjob
The problem seems to be related to switching from
prp_small_pooltoprp_page_pool; the former is optimized for I/O between 4k and 128k.Quoted 6 lines
Expanding the small pool size to cover up to 256k increases the performance. I'm not sure however if this is the proper and general fix. For one thing, expanding the pool size bumps the numbers for a short burst test (10 sec) but the numbers drop again significantly during a longer test. The behavior is not unique to
io_uringeither. Included below are a couple of 256k fio pvsync2 tests as well.Quoted 6 lines
I am surprised you're seeing such a drop just from the prp pool used.
What CPU architecture are you using? Reason I ask: the driver allocates
PAGE_SIZEfor the large prp pool, but we really wantctrl->page_size(always 4k). If your CPU architecture has aPAGE_SIZElarger than 4k, could you try the following?It's an
x86_64vm with 8GB of memory. Is the 4k pool size meant to support up to a 2MB i/o size then?
Quoted 6 lines
The main seems to be a lock contention, this what I see with the 256k test after running a while. However, I can't pinpoint the lock with perf lock; is there a better way to do that?
65.08% [kernel] [k] __pv_queued_spin_lock_slowpath 2.43% [kernel] [k] mutex_spin_on_owner
It should be figured out by 'perf record -g'.
Quoted 9 lines
Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian CPU(s): 16 On-line CPU(s) list: 0-15 Thread(s) per core: 1 Core(s) per socket: 1 Socket(s): 16
Looks processor emulation is weird, it said you have 16 sockets, and each socket has just one CPU core. Maybe this is the real problem.
Thanks,
Ming Lei
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 2 lines · Bijan Mottahedeh
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
Try using a controller with SGL support. NVMe PRP unfortunately are a completely brain dead scheme once you use non-tiny I/O sizes ( and actually really painful even for those for other reasons).
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Keith Busch
maintainerQuoted 7 lines · Christoph Hellwig
Quoted 2 lines · Bijan Mottahedeh
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
Try using a controller with SGL support. NVMe PRP unfortunately are a completely brain dead scheme once you use non-tiny I/O sizes ( and actually really painful even for those for other reasons).
SGL would really help if you have phyiscal continuity, and never needs to allocate from the dma pools. If most pages in the transfer are not physically contiguous, though, PRP is still more memory efficient. But yes, the PRP format is ... unique. :)
FWIW, I couldn't measure a performance drop on real hardware comparing the large pool vs the small one.
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 8 lines · Keith Busch
Quoted 3 lines
Quoted 3 lines
Try using a controller with SGL support. NVMe PRP unfortunately are a completely brain dead scheme once you use non-tiny I/O sizes ( and actually really painful even for those for other reasons).
SGL would really help if you have phyiscal continuity, and never needs to allocate from the dma pools. If most pages in the transfer are not physically contiguous, though, PRP is still more memory efficient. But yes, the PRP format is ... unique.:)
When we added SGL support I remember having ~5% increase with the controller which was supporting SGLs with fio block size > 32K, also there was performance drop when SGL is used for block size < 32k (i.e. 4k,8k,16k).
Maybe it is worth taking numbers on the controller with varying block size with PRPs vs SGLs (4k, 8k, 16k, 32k, 64k) with just random-read to avoid any gc effect ?
Quoted 2 lines
FWIW, I couldn't measure a performance drop on real hardware comparing the large pool vs the small one.
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 11 lines · Keith Busch
Quoted 6 lines · Christoph Hellwig
Quoted 2 lines · Bijan Mottahedeh
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
Try using a controller with SGL support. NVMe PRP unfortunately are a completely brain dead scheme once you use non-tiny I/O sizes ( and actually really painful even for those for other reasons).
SGL would really help if you have phyiscal continuity, and never needs to allocate from the dma pools. If most pages in the transfer are not physically contiguous, though, PRP is still more memory efficient. But yes, the PRP format is ... unique. :)
So with SGL you can potentially bypass dma pool allocation but with PRP you always have to regardless of contiguity?
Quoted 3 lines
FWIW, I couldn't measure a performance drop on real hardware comparing the large pool vs the small one.
I took out my workaround.
I can't see a big drop on h/w with 256k either; however, I can't push the tests far enough because they hang depending on the number of fio jobs and iodepth.
They seem to be stuck in an io_uring poll loop:
58.91% [kernel] [k] blk_poll 27.01% [kernel] [k] io_iopoll_getevents 8.56% [kernel] [k] blkdev_iopoll
I had to kill the fio jobs and got a bunch of this output:
Jobs: 103 (f=103): [r(4),_(1),r(12),_(1),r(2),_(1),r(10),_(1),r(2),_(1),r(1),_(1),r(7),_(1),r(1),_(2),r(16),_(1),r(5),_(1),r(6),_(2),r(2),_(1),r(2),_(1),r(2),_(1),r(3),_(1),r(9),_(1),r(4),_(2),r(5),_(1),r(3),_(1),r(1),_(1),r(4),_(2),r(2)][0fio: job 'fiotest' (state=5) hasn't exited in 300 seconds, it appears to be stuck. Doing forceful exit of this job.
Hopefully Jens can comment what this means.
--bijan
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Keith Busch
maintainerQuoted 8 lines · Bijan Mottahedeh
Quoted 7 lines · Keith Busch
Quoted 6 lines · Christoph Hellwig
Quoted 2 lines · Bijan Mottahedeh
I'm seeing a sizeable drop in perf with polled fio tests for block sizes > 128k:
Try using a controller with SGL support. NVMe PRP unfortunately are a completely brain dead scheme once you use non-tiny I/O sizes ( and actually really painful even for those for other reasons).
Quoted 7 lines
Quoted 4 lines
SGL would really help if you have phyiscal continuity, and never needs to allocate from the dma pools. If most pages in the transfer are not physically contiguous, though, PRP is still more memory efficient. But yes, the PRP format is ... unique. :)
So with SGL you can potentially bypass dma pool allocation but with PRP you always have to regardless of contiguity?
Yes, if the payload can be expressed as a single address range, an SGL capable controller can describe that in the NVMe command without allocating a scatter list payload to accompany the command.
The maximum length of a PRP is 4k. If you're transferring 256k per command, that will require at least 64 PRP entries regardless of the number of ranges in the DMA mapped scatter-list.
Quoted 9 lines
Quoted 2 lines
FWIW, I couldn't measure a performance drop on real hardware comparing the large pool vs the small one.
I took out my workaround.
I can't see a big drop on h/w with 256k either; however, I can't push the tests far enough because they hang depending on the number of fio jobs and iodepth.
Quoted 5 lines
They seem to be stuck in an
io_uringpoll loop:58.91% [kernel] [k]
blk_poll27.01% [kernel] [k]io_iopoll_getevents8.56% [kernel] [k]blkdev_iopoll
That looks pretty normal for polled io.
Quoted 7 lines
I had to kill the fio jobs and got a bunch of this output:
Jobs: 103 (f=103): [r(4),_(1),r(12),_(1),r(2),_(1),r(10),_(1),r(2),_(1),r(1),_(1),r(7),_(1),r(1),_(2),r(16),_(1),r(5),_(1),r(6),_(2),r(2),_(1),r(2),_(1),r(2),_(1),r(3),_(1),r(9),_(1),r(4),_(2),r(5),_(1),r(3),_(1),r(1),_(1),r(4),_(2),r(2)][0fio:
job 'fiotest' (state=5) hasn't exited in 300 seconds, it appears to be
stuck. Doing forceful exit of this job.Hopefully Jens can comment what this means.
What kernel are you using? A bug observed with multiple threads polling the same queue was just fixed in the 5.6-rc5 release (also in 5.5.8, 5.4.24 stables).
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Keith Busch
maintainerQuoted 11 lines · Keith Busch
Quoted 7 lines · Bijan Mottahedeh
I had to kill the fio jobs and got a bunch of this output:
Jobs: 103 (f=103): [r(4),_(1),r(12),_(1),r(2),_(1),r(10),_(1),r(2),_(1),r(1),_(1),r(7),_(1),r(1),_(2),r(16),_(1),r(5),_(1),r(6),_(2),r(2),_(1),r(2),_(1),r(2),_(1),r(3),_(1),r(9),_(1),r(4),_(2),r(5),_(1),r(3),_(1),r(1),_(1),r(4),_(2),r(2)][0fio:
job 'fiotest' (state=5) hasn't exited in 300 seconds, it appears to be
stuck. Doing forceful exit of this job.Hopefully Jens can comment what this means.
What kernel are you using? A bug observed with multiple threads polling the same queue was just fixed in the 5.6-rc5 release
Oops, I suppose you already know that ... you fixed it! :)
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme