nvme-tcp: improve scalability
v2 cover nvme-tcp: improve scalability
Hi all,
for workloads with a lot of controllers we run into workqueue contention, where the single workqueue is not able to service requests fast enough, leading to spurious I/O errors and connect resets during high load. This patchset improves the situation by improve the fairness between rx and tx scheduling, introducing per-controller workqueues, and distribute the load accoring to the blk-mq cpu mapping. With this we reduce the spurious I/O errors and improve the overall performance for highly contended workloads.
All performance number are derived from the 'tiobench-example.fio' sample from the fio sources, running on a 96 core machine with one subsystem and two paths, each path exposing 32 queues. Backend is nvmet using an Intel DC P3700 NVMe SSD.
Changes to the initial submission:
- Make the changes independent from the 'wq_unbound' parameter
- Drop changes to the workqueue
- Add patch to improve rx/tx fairness
Hannes Reinecke (3):
nvme-tcp: improve rx/tx fairness
nvme-tcp: align I/O cpu with blk-mq mapping
nvme-tcp: per-controller I/O workqueues
drivers/nvme/host/tcp.c | 135 ++++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 40 deletions(-)
Signature · 1 line
2.35.3
PATCH v2 1/3 nvme-tcp: improve rx/tx fairness
We need to restrict both side, rx and tx, to only run for a certain time to ensure that we're not blocking the other side and induce starvation. So pass in a 'deadline' value to nvme_tcp_send_all() and nvme_tcp_try_recv() and break out of the loop if the deadline is reached.
As we now have a timestamp we can also use it to print out a warning if the actual time spent exceeds the deadline.
Performance comparison:
baseline rx/tx fairness
4k seq write: 449MiB/s 480MiB/s
4k rand write: 410MiB/s 481MiB/s
4k seq read: 478MiB/s 481MiB/s
4k rand read: 547MiB/s 480MiB/s
Random read is ever so disappointing, but that will be fixed with the later patches.
--- drivers/nvme/host/tcp.c | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-)
drivers/nvme/host/tcp.c +29 -9
@@ -153,6 +153,7 @@ struct nvme_tcp_queue {size_t data_remaining;size_t ddgst_remaining;unsigned int nr_cqe; +unsigned long deadline;/* send state */ struct nvme_tcp_request *request;
@@ -359,14 +360,18 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, } } -static inline void nvme_tcp_send_all (struct nvme_tcp_queue *queue) +static inline int nvme_tcp_send_all (struct nvme_tcp_queue *queue, +unsigned long deadline) {int ret;/* drain the send queue as much as we can... */ do { ret =nvme_tcp_try_send (queue); +if (time_after (jiffies, deadline)) +break ; }while (ret >0 ); +return ret; }static inline bool nvme_tcp_queue_has_pending (struct nvme_tcp_queue *queue)
@@ -385,6 +390,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, bool sync, bool last) {struct nvme_tcp_queue *queue = req->queue; +unsigned long deadline = jiffies +msecs_to_jiffies (1 );bool empty; empty = llist_add(&req->lentry, &queue->req_list) &&
@@ -397,7 +403,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,*/ if (queue->io_cpu ==raw_smp_processor_id () && sync && empty &&mutex_trylock (&queue->send_mutex)) { -nvme_tcp_send_all (queue); +nvme_tcp_send_all (queue, deadline);mutex_unlock (&queue->send_mutex); }
@@ -959,9 +965,14 @@ static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,nvme_tcp_error_recovery (&queue->ctrl->ctrl);return result; } +if (time_after (jiffies, queue->deadline)) { + desc->count =0 ; +break ; + } + } -return consumed; +return consumed - len; }static void nvme_tcp_data_ready (struct sock *sk)
@@ -1258,7 +1269,7 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)return ret; } -static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue) +static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue,unsigned long deadline) {struct socket *sock = queue->sock;struct sock *sk = sock->sk;
@@ -1269,6 +1280,7 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) rd_desc.count =1 ;lock_sock (sk); queue->nr_cqe =0 ; + queue->deadline = deadline; consumed = sock->ops->read_sock (sk, &rd_desc, nvme_tcp_recv_skb);release_sock (sk);return consumed;
@@ -1278,14 +1290,15 @@ static void nvme_tcp_io_work(struct work_struct *w) {struct nvme_tcp_queue *queue = container_of(w, struct nvme_tcp_queue, io_work); -unsigned long deadline = jiffies +msecs_to_jiffies (1 ); +unsigned long tx_deadline = jiffies +msecs_to_jiffies (1 ); +unsigned long rx_deadline = tx_deadline +msecs_to_jiffies (1 ), overrun; do { bool pending = false; int result; if (mutex_trylock(&queue->send_mutex)) { - result = nvme_tcp_try_send(queue); + result = nvme_tcp_send_all(queue, tx_deadline); mutex_unlock(&queue->send_mutex); if (result > 0) pending = true;
@@ -1293,7 +1306,7 @@ static void nvme_tcp_io_work(struct work_struct *w)break ; } - result =nvme_tcp_try_recv (queue); + result =nvme_tcp_try_recv (queue, rx_deadline);if (result >0 ) pending = true;else if (unlikely (result <0 ))
@@ -1302,7 +1315,13 @@ static void nvme_tcp_io_work(struct work_struct *w)if (!pending || !queue->rd_enabled)return ; - }while (!time_after (jiffies, deadline));/* quota is exhausted */ + }while (!time_after (jiffies, rx_deadline));/* quota is exhausted */ + + overrun = jiffies - rx_deadline; +if (nvme_tcp_queue_id (queue) >0 && + overrun >msecs_to_jiffies (10 )) +dev_dbg (queue->ctrl->ctrl.device,"queue %d: queue stall (%u msecs)\n" , +nvme_tcp_queue_id (queue),jiffies_to_msecs (overrun));queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); }
@@ -2666,6 +2685,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) {struct nvme_tcp_queue *queue = hctx->driver_data;struct sock *sk = queue->sock->sk; +unsigned long deadline = jiffies +msecs_to_jiffies (1 );if (!test_bit (NVME_TCP_Q_LIVE , &queue->flags))return 0 ;
@@ -2673,7 +2693,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)set_bit (NVME_TCP_Q_POLLING , &queue->flags);if (sk_can_busy_loop (sk) &&skb_queue_empty_lockless (&sk->sk_receive_queue))sk_busy_loop (sk, true); -nvme_tcp_try_recv (queue); +nvme_tcp_try_recv (queue, deadline);clear_bit (NVME_TCP_Q_POLLING , &queue->flags);return queue->nr_cqe; }
Signature · 1 line
2.35.3
Sagi Grimberg
maintainerRe: [PATCH 1/3] nvme-tcp: improve rx/tx fairness
Hey Hannes, thanks for doing this.
Quoted 3 lines · Hannes Reinecke
We need to restrict both side, rx and tx, to only run for a certain time to ensure that we're not blocking the other side and induce starvation. So pass in a 'deadline' value to
nvme_tcp_send_all()
Please split the addition of nvme_tcp_send_all() to a separate prep patch.
Quoted 2 lines
and
nvme_tcp_try_recv()
and break out of the loop if the deadline is reached.
I think we want to limit the rx/tx in pdus/bytes. This will also allow us to possibly do burst rx from data-ready.
Quoted 11 lines
As we now have a timestamp we can also use it to print out a warning if the actual time spent exceeds the deadline.
Performance comparison:
baseline rx/tx fairness
4k seq write: 449MiB/s 480MiB/s
4k rand write: 410MiB/s 481MiB/s
4k seq read: 478MiB/s 481MiB/s
4k rand read: 547MiB/s 480MiB/s
Quoted 2 lines
Random read is ever so disappointing, but that will be fixed with the later patches.
That is a significant decline in relative perf. I'm counting 12.5%... Can you explain why that is?
How does this look for multiple controllers?
Quoted 11 lines
Signed-off-by Hannes Reinecke <[email protected]>--- drivers/nvme/host/tcp.c | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-)drivers/nvme/host/tcp.c +0
-0
drivers/nvme/host/tcp.c struct nvme_tcp_queue { · line 156
Quoted 4 lines
drivers/nvme/host/tcp.c +1
-0@@ -153,6 +153,7 @@ struct nvme_tcp_queue {size_t data_remaining;size_t ddgst_remaining;unsigned int nr_cqe; +unsigned long deadline;
I don't see why you need to keep this in the queue struct. You could have easily initialize it in the read_descriptor_t and test against it.
Quoted 10 lines
drivers/nvme/host/tcp.c +2
-1/* send state */ struct nvme_tcp_request *request;@@ -359,14 +360,18 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, } } -static inline void nvme_tcp_send_all (struct nvme_tcp_queue *queue) +static inline int nvme_tcp_send_all (struct nvme_tcp_queue *queue, +unsigned long deadline)
drivers/nvme/host/tcp.c static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, · line 374
Quoted 10 lines
drivers/nvme/host/tcp.c +3
-0{int ret;/* drain the send queue as much as we can... */ do { ret =nvme_tcp_try_send (queue); +if (time_after (jiffies, deadline)) +break ; }while (ret >0 ); +return ret;
I think you want a different interface, nvme_tcp_send_budgeted(queue, budget). I don't know what you pass here, but jiffies is a rather large granularity...
Quoted 12 lines
drivers/nvme/host/tcp.c +1
-0}static inline bool nvme_tcp_queue_has_pending (struct nvme_tcp_queue *queue)@@ -385,6 +390,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, bool sync, bool last) {struct nvme_tcp_queue *queue = req->queue; +unsigned long deadline = jiffies +msecs_to_jiffies (1 );bool empty; empty = llist_add(&req->lentry, &queue->req_list) &&
drivers/nvme/host/tcp.c static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, · line 408
Quoted 7 lines
drivers/nvme/host/tcp.c +1
-1@@ -397,7 +403,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,*/ if (queue->io_cpu ==raw_smp_processor_id () && sync && empty &&mutex_trylock (&queue->send_mutex)) { -nvme_tcp_send_all (queue); +nvme_tcp_send_all (queue, deadline);mutex_unlock (&queue->send_mutex); }
Umm, spend up to a millisecond in in queue_request ? Sounds like way too much... Did you ever see this deadline exceeded? sends should be rather quick...
drivers/nvme/host/tcp.c static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, · line 972
Quoted 10 lines
drivers/nvme/host/tcp.c +5
-0@@ -959,9 +965,14 @@ static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,nvme_tcp_error_recovery (&queue->ctrl->ctrl);return result; } +if (time_after (jiffies, queue->deadline)) { + desc->count =0 ; +break ; + } +
That is still not right.
You don't want to spend a full deadline reading from the socket, and then spend a full deadline writing to the socket...
You want the io_work to take a full deadline, and send budgets of try_send and try_recv. And set it to sane counts. Say 8 pdus, or 64k bytes. We want to get to some magic value that presents a sane behavior, that confidently fits inside a deadline, and is fair.
Quoted 43 lines
drivers/nvme/host/tcp.c +6
-4} -return consumed; +return consumed - len; }static void nvme_tcp_data_ready (struct sock *sk)@@ -1258,7 +1269,7 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)return ret; } -static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue) +static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue,unsigned long deadline) {struct socket *sock = queue->sock;struct sock *sk = sock->sk;@@ -1269,6 +1280,7 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) rd_desc.count =1 ;lock_sock (sk); queue->nr_cqe =0 ; + queue->deadline = deadline; consumed = sock->ops->read_sock (sk, &rd_desc, nvme_tcp_recv_skb);release_sock (sk);return consumed;@@ -1278,14 +1290,15 @@ static void nvme_tcp_io_work(struct work_struct *w) {struct nvme_tcp_queue *queue = container_of(w, struct nvme_tcp_queue, io_work); -unsigned long deadline = jiffies +msecs_to_jiffies (1 ); +unsigned long tx_deadline = jiffies +msecs_to_jiffies (1 ); +unsigned long rx_deadline = tx_deadline +msecs_to_jiffies (1 ), overrun; do { bool pending = false; int result; if (mutex_trylock(&queue->send_mutex)) { - result = nvme_tcp_try_send(queue); + result = nvme_tcp_send_all(queue, tx_deadline); mutex_unlock(&queue->send_mutex); if (result > 0) pending = true;
drivers/nvme/host/tcp.c static void nvme_tcp_io_work(struct work_struct *w) · line 1309
Quoted 5 lines
drivers/nvme/host/tcp.c +1
-1@@ -1293,7 +1306,7 @@ static void nvme_tcp_io_work(struct work_struct *w)break ; } - result =nvme_tcp_try_recv (queue); + result =nvme_tcp_try_recv (queue, rx_deadline);
I think you want a more frequent substitution of sends/receives. the granularity of 1ms budget may be too coarse?
Quoted 5 lines
drivers/nvme/host/tcp.c +0
-0if (result >0 ) pending = true;else if (unlikely (result <0 ))@@ -1302,7 +1315,13 @@ static void nvme_tcp_io_work(struct work_struct *w)if (!pending || !queue->rd_enabled)
drivers/nvme/host/tcp.c static void nvme_tcp_io_work(struct work_struct *w) · line 1324
Quoted 10 lines
drivers/nvme/host/tcp.c +7
-1return ; - }while (!time_after (jiffies, deadline));/* quota is exhausted */ + }while (!time_after (jiffies, rx_deadline));/* quota is exhausted */ + + overrun = jiffies - rx_deadline; +if (nvme_tcp_queue_id (queue) >0 && + overrun >msecs_to_jiffies (10 )) +dev_dbg (queue->ctrl->ctrl.device,"queue %d: queue stall (%u msecs)\n" , +nvme_tcp_queue_id (queue),jiffies_to_msecs (overrun));
Umm, ok. why 10? why not 2? or 3?
Do you expect io_work to spend more time executing?
Quoted 12 lines
drivers/nvme/host/tcp.c +1
-0queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); }@@ -2666,6 +2685,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) {struct nvme_tcp_queue *queue = hctx->driver_data;struct sock *sk = queue->sock->sk; +unsigned long deadline = jiffies +msecs_to_jiffies (1 );if (!test_bit (NVME_TCP_Q_LIVE , &queue->flags))return 0 ;
drivers/nvme/host/tcp.c static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) · line 2696
Quoted 5 lines
drivers/nvme/host/tcp.c +1
-1@@ -2673,7 +2693,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)set_bit (NVME_TCP_Q_POLLING , &queue->flags);if (sk_can_busy_loop (sk) &&skb_queue_empty_lockless (&sk->sk_receive_queue))sk_busy_loop (sk, true); -nvme_tcp_try_recv (queue); +nvme_tcp_try_recv (queue, deadline);
spend a millisecond in nvme_tcp_poll() ??
Isn't it too long?
Re: [PATCH 1/3] nvme-tcp: improve rx/tx fairness
Quoted 9 lines · Sagi Grimberg
Hey Hannes, thanks for doing this.
Quoted 3 lines · Hannes Reinecke
We need to restrict both side, rx and tx, to only run for a certain time to ensure that we're not blocking the other side and induce starvation. So pass in a 'deadline' value to
nvme_tcp_send_all()Please split the addition of
nvme_tcp_send_all()to a separate prep patch.
Okay, no problem.
Quoted 6 lines
Quoted 2 lines
and
nvme_tcp_try_recv()
and break out of the loop if the deadline is reached.I think we want to limit the rx/tx in pdus/bytes. This will also allow us to possibly do burst rx from data-ready.
PDUs is not the best scheduling boundary here, as each PDU can be of different size, and the network interface most definitely is limited by the number of bytes transferred.
Quoted 11 lines
Quoted 11 lines
As we now have a timestamp we can also use it to print out a warning if the actual time spent exceeds the deadline.
Performance comparison:
baseline rx/tx fairness
4k seq write: 449MiB/s 480MiB/s
4k rand write: 410MiB/s 481MiB/s
4k seq read: 478MiB/s 481MiB/s
4k rand read: 547MiB/s 480MiB/s
Quoted 6 lines
Quoted 2 lines
Random read is ever so disappointing, but that will be fixed with the later patches.
That is a significant decline in relative perf. I'm counting 12.5%... Can you explain why that is?
Not really :-(
But then fairness cuts both ways; so I am not surprised that some workloads suffer here.
Quoted 2 lines
How does this look for multiple controllers?
Haven't really checked (yet); it would make a rather weak case if we killed performance just to scale better ...
Quoted 8 lines
Quoted 8 lines
Signed-off-by Hannes Reinecke <[email protected]>---
drivers/nvme/host/tcp.c| 38 +++++++++++++++++++++++++++++---------1 file changed, 29 insertions(+), 9 deletions(-)
Quoted 13 lines
drivers/nvme/host/tcp.c struct nvme_tcp_queue { · line 156
Quoted 9 lines
drivers/nvme/host/tcp.c +1
-0@@ -153,6 +153,7 @@ struct nvme_tcp_queue { size_t data_remaining; size_t ddgst_remaining; unsigned int nr_cqe; + unsigned long deadline;I don't see why you need to keep this in the queue struct. You could have easily initialize it in the
read_descriptor_tand test against it.
Because I wanted to interrupt the receive side for large data transfers, and haven't found a way to pass the deadline into ->read_sock().
Quoted 13 lines
Quoted 13 lines
drivers/nvme/host/tcp.c +2
-1/* send state */ struct nvme_tcp_request *request;@@ -359,14 +360,18 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, } } -static inline void nvme_tcp_send_all (struct nvme_tcp_queue *queue) +static inline int nvme_tcp_send_all (struct nvme_tcp_queue *queue, + unsigned long deadline) { int ret; /* drain the send queue as much as we can... */ do {
Quoted 9 lines
drivers/nvme/host/tcp.c static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, · line 374
Quoted 5 lines
drivers/nvme/host/tcp.c +3
-0ret = nvme_tcp_try_send(queue); + if (time_after(jiffies, deadline)) + break; } while (ret > 0); + return ret;I think you want a different interface,
nvme_tcp_send_budgeted(queue, budget). I don't know what you pass here, but jiffies is a rather large granularity...
Hmm. I've been using jiffies as the io_work loop had been counting in jiffies. But let me check what would happen if I move that over to PDU / size counting.
Quoted 14 lines
Quoted 14 lines
drivers/nvme/host/tcp.c +1
-0} static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue)@@ -385,6 +390,7 @@ static inline void nvme_tcp_queue_request(structnvme_tcp_request *req, bool sync, bool last) { struct nvme_tcp_queue *queue = req->queue; + unsigned long deadline = jiffies + msecs_to_jiffies(1); bool empty;empty =
llist_add(&req->lentry, &queue->req_list)&&drivers/nvme/host/tcp.c +0
-0@@ -397,7 +403,7 @@ static inline void nvme_tcp_queue_request(structnvme_tcp_request *req, */ if (
queue->io_cpu==raw_smp_processor_id()&&
Quoted 9 lines
Quoted 5 lines
sync && empty &&
mutex_trylock(&queue->send_mutex)) { -nvme_tcp_send_all(queue);
+nvme_tcp_send_all(queue, deadline);mutex_unlock(&queue->send_mutex);
}Umm, spend up to a millisecond in in
queue_request? Sounds like way too much... Did you ever see this deadline exceeded? sends should be rather quick...
Of _course_ it's too long. That's kinda the point.
But I'm seeing network latency up to 4000 msecs (!) on my test setup, so _that_ is the least of my worries ...
Quoted 5 lines
Quoted 5 lines
drivers/nvme/host/tcp.c +0
-0@@ -959,9 +965,14 @@ static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, nvme_tcp_error_recovery(&queue->ctrl->ctrl); return result; }
Quoted 9 lines
drivers/nvme/host/tcp.c static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, · line 972
Quoted 5 lines
drivers/nvme/host/tcp.c +5
-0+ if (time_after(jiffies, queue->deadline)) { + desc->count = 0; + break; + } +That is still not right.
You don't want to spend a full deadline reading from the socket, and then spend a full deadline writing to the socket...
Yes, and no.
Problem is that the current code serializes writes and reads. Essentially for each iteration we first to a write, and then a read. If we spend the full deadline on write we will need to reschedule, but we then _again_ start with writes. This leads to a heavy preference for writing, and negative performance impact.
Quoted 4 lines
You want the
io_workto take a full deadline, and send budgets oftry_sendandtry_recv. And set it to sane counts. Say 8 pdus, or 64k bytes. We want to get to some magic value that presents a
sane behavior, that confidently fits inside a deadline, and is fair.
Easier said than done.
Biggest problem is that most of the latency increase comes from the actual 'sendmsg()' and 'read_sock()' calls.
And the only way of inhibiting that would be to check _prior_ whether we can issue the call in the first place.
(That's why I did the SOCK_NOSPACE tests in the previous patchsets).
Quoted 41 lines
Quoted 41 lines
drivers/nvme/host/tcp.c +2
-2} - return consumed; + return consumed - len; } static void nvme_tcp_data_ready(struct sock *sk)@@ -1258,7 +1269,7 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) return ret; } -static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue) +static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue,unsigned long deadline) { struct socket *sock = queue->sock;struct sock *sk =
sock->sk;drivers/nvme/host/tcp.c +1
-0@@ -1269,6 +1280,7 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) rd_desc.count = 1; lock_sock(sk); queue->nr_cqe = 0; + queue->deadline = deadline; consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); release_sock(sk);return consumed;
drivers/nvme/host/tcp.c +0
-0@@ -1278,14 +1290,15 @@ static void nvme_tcp_io_work(struct work_struct *w) {struct
nvme_tcp_queue*queue =
container_of(w, struct nvme_tcp_queue, io_work); - unsigned long deadline = jiffies +msecs_to_jiffies(1); + unsigned longtx_deadline= jiffies +msecs_to_jiffies(1); + unsigned longrx_deadline=tx_deadline+msecs_to_jiffies(1), overrun;
do {
bool pending = false;
int result;
if (mutex_trylock(&queue->send_mutex)) {
- result =nvme_tcp_try_send(queue);
+ result =nvme_tcp_send_all(queue, tx_deadline);mutex_unlock(&queue->send_mutex);
if (result > 0)
Quoted 9 lines
drivers/nvme/host/tcp.c static void nvme_tcp_io_work(struct work_struct *w) · line 1308
Quoted 6 lines
pending = true;
drivers/nvme/host/tcp.c +1
-1@@ -1293,7 +1306,7 @@ static void nvme_tcp_io_work(struct work_struct *w) break; } - result = nvme_tcp_try_recv(queue); + result = nvme_tcp_try_recv(queue, rx_deadline);I think you want a more frequent substitution of sends/receives. the granularity of 1ms budget may be too coarse?
The problem is not the granularity, the problem is the latency spikes I'm seeing when issuing 'sendmsg' or 'read_sock'.
Quoted 12 lines
Quoted 12 lines
drivers/nvme/host/tcp.c +1
-1if (result > 0) pending = true; else if (unlikely(result < 0))@@ -1302,7 +1315,13 @@ static void nvme_tcp_io_work(struct work_struct *w) if (!pending || !queue->rd_enabled) return; - } while (!time_after(jiffies, deadline)); /* quota is exhausted */ + } while (!time_after(jiffies, rx_deadline)); /* quota isexhausted */
+
+ overrun = jiffies -rx_deadline;
Quoted 9 lines
Quoted 5 lines
+ if (
nvme_tcp_queue_id(queue)> 0 &&
+ overrun >msecs_to_jiffies(10))
+ dev_dbg(queue->ctrl->ctrl.device, "queue %d: queue stall (%u msecs)\n",
+nvme_tcp_queue_id(queue),jiffies_to_msecs(overrun));Umm, ok. why 10? why not 2? or 3?
Do you expectio_workto spend more time executing?
Eg 4000 msecs like on my testbed?
Yes.
Quoted 12 lines
Quoted 12 lines
queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); }drivers/nvme/host/tcp.c +1
-0@@ -2666,6 +2685,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { struct nvme_tcp_queue *queue = hctx->driver_data; struct sock *sk = queue->sock->sk; + unsigned long deadline = jiffies + msecs_to_jiffies(1); if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) return 0;@@ -2673,7 +2693,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
Quoted 8 lines
drivers/nvme/host/tcp.c static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) · line 2696
Quoted 5 lines
drivers/nvme/host/tcp.c +1
-1set_bit(NVME_TCP_Q_POLLING, &queue->flags); if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) sk_busy_loop(sk, true); - nvme_tcp_try_recv(queue); + nvme_tcp_try_recv(queue, deadline);spend a millisecond in
nvme_tcp_poll()??
Isn't it too long?
Haven't tried with polling, so can't honestly answer.
In the end, it all boils down to numbers.
I'm having 2 controllers with 32 queues and 256 requests. Running on a 10GigE link one should be getting a net throughput of 1GB/s, or, with 4k requests, 256k IOPS.
Or a latency of 0.25 milliseconds per command.
In the optimal case. As I'm seeing a bandwidth of around 500MB/s, I'm looking at a latency 0.5 milliseconds _in the ideal case_.
So no, I don't think the 1 milliseconds is too long.
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Sagi Grimberg
maintainerRe: [PATCH 1/3] nvme-tcp: improve rx/tx fairness
Quoted 12 lines · Hannes Reinecke
Quoted 9 lines · Sagi Grimberg
Hey Hannes, thanks for doing this.
Quoted 3 lines · Hannes Reinecke
We need to restrict both side, rx and tx, to only run for a certain time to ensure that we're not blocking the other side and induce starvation. So pass in a 'deadline' value to
nvme_tcp_send_all()Please split the addition of
nvme_tcp_send_all()to a separate prep patch.Okay, no problem.
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
and
nvme_tcp_try_recv()
and break out of the loop if the deadline is reached.I think we want to limit the rx/tx in pdus/bytes. This will also allow us to possibly do burst rx from data-ready.
PDUs is not the best scheduling boundary here, as each PDU can be of different size, and the network interface most definitely is limited by the number of bytes transferred.
Well, PDUs is at least A proxy of bytes in terms of limits, but I agree bytes may be more appropriate.
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
As we now have a timestamp we can also use it to print out a warning if the actual time spent exceeds the deadline.
Performance comparison:
baseline rx/tx fairness
4k seq write: 449MiB/s 480MiB/s
4k rand write: 410MiB/s 481MiB/s
4k seq read: 478MiB/s 481MiB/s
4k rand read: 547MiB/s 480MiB/s
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
Random read is ever so disappointing, but that will be fixed with the later patches.
That is a significant decline in relative perf. I'm counting 12.5%... Can you explain why that is?
Not really :-(
But then fairness cuts both ways; so I am not surprised that some workloads suffer here.
Well, 12.5% is rather steep, don't you agree?
I'm even more surprised that seq/rand read make a difference...
Quoted 28 lines
Quoted 4 lines
How does this look for multiple controllers?
Haven't really checked (yet); it would make a rather weak case if we killed performance just to scale better ...
Quoted 8 lines
Quoted 8 lines
Signed-off-by Hannes Reinecke <[email protected]>---
drivers/nvme/host/tcp.c| 38 +++++++++++++++++++++++++++++---------1 file changed, 29 insertions(+), 9 deletions(-)Quoted 13 lines
drivers/nvme/host/tcp.c struct nvme_tcp_queue { · line 156
Quoted 9 lines
drivers/nvme/host/tcp.c +1
-0@@ -153,6 +153,7 @@ struct nvme_tcp_queue { size_t data_remaining; size_t ddgst_remaining; unsigned int nr_cqe; + unsigned long deadline;I don't see why you need to keep this in the queue struct. You could have easily initialize it in the
read_descriptor_tand test against it.
Quoted 2 lines
Because I wanted to interrupt the receive side for large data transfers, and haven't found a way to pass the deadline into ->read_sock().
desc.count is for you to interpret however you want it to be no?
Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
drivers/nvme/host/tcp.c +2
-1/* send state */ struct nvme_tcp_request *request;@@ -359,14 +360,18 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, } } -static inline void nvme_tcp_send_all (struct nvme_tcp_queue *queue) +static inline int nvme_tcp_send_all (struct nvme_tcp_queue *queue, + unsigned long deadline) { int ret; /* drain the send queue as much as we can... */ do {Quoted 9 lines
drivers/nvme/host/tcp.c static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, · line 374
Quoted 5 lines
drivers/nvme/host/tcp.c +3
-0ret = nvme_tcp_try_send(queue); + if (time_after(jiffies, deadline)) + break; } while (ret > 0); + return ret;I think you want a different interface,
nvme_tcp_send_budgeted(queue, budget). I don't know what you pass here, but jiffies is a rather large granularity...
Quoted 3 lines
Hmm. I've been using jiffies as the
io_workloop had been counting in jiffies. But let me check what would happen if I move that over to PDU / size counting.
Cool.
Quoted 24 lines
Quoted 15 lines
Quoted 15 lines
drivers/nvme/host/tcp.c +1
-0} static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue)@@ -385,6 +390,7 @@ static inline void nvme_tcp_queue_request(structnvme_tcp_request *req, bool sync, bool last) { struct nvme_tcp_queue *queue = req->queue; + unsigned long deadline = jiffies + msecs_to_jiffies(1); bool empty;empty =
llist_add(&req->lentry, &queue->req_list)&&drivers/nvme/host/tcp.c +0
-0@@ -397,7 +403,7 @@ static inline void nvme_tcp_queue_request(structnvme_tcp_request *req, */ if (
queue->io_cpu==raw_smp_processor_id()&&Quoted 9 lines
Quoted 5 lines
sync && empty &&
mutex_trylock(&queue->send_mutex)) { -nvme_tcp_send_all(queue);
+nvme_tcp_send_all(queue, deadline);mutex_unlock(&queue->send_mutex);
}Umm, spend up to a millisecond in in
queue_request? Sounds like way too much... Did you ever see this deadline exceeded? sends should be rather quick...
Quoted 3 lines
Of _course_ it's too long. That's kinda the point.
But I'm seeing network latency up to 4000 msecs (!) on my test setup, so _that_ is the least of my worries ...
This has nothing to do with the network latency afaict. This is just a matter of how long does it take to write to the socket buffer (or in our case, construct skbs and add them to the socket as TX is zero copy).
Quoted 15 lines
Quoted 15 lines
drivers/nvme/host/tcp.c static int nvme_tcp_recv_skb(read_descriptor_t · line 973
Quoted 11 lines
drivers/nvme/host/tcp.c +5
-0@@ -959,9 +965,14 @@ static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,nvme_tcp_error_recovery (&queue->ctrl->ctrl); return result; } + if (time_after(jiffies, queue->deadline)) { + desc->count = 0; + break; + } +That is still not right.
You don't want to spend a full deadline reading from the socket, and then spend a full deadline writing to the socket...
Quoted 6 lines
Yes, and no.
Problem is that the current code serializes writes and reads. Essentially for each iteration we first to a write, and then a read. If we spend the full deadline on write we will need to reschedule, but we then _again_ start with writes. This leads to a heavy preference for writing, and negative performance impact.
Yes, the assumption we should take is that the tx/rx limits allow to switch between the two at least once or twice in the io_work while loop, and that they get a fair'ish share of quota.
Quoted 8 lines
Quoted 5 lines
You want the
io_workto take a full deadline, and send budgets oftry_sendandtry_recv. And set it to sane counts. Say 8 pdus, or 64k bytes. We want to get to some magic value that presents a
sane behavior, that confidently fits inside a deadline, and is fair.Easier said than done.
Biggest problem is that most of the latency increase comes from the actual 'sendmsg()' and 'read_sock()' calls.
This is a bit surprising. because read_sock reads data that is already processed by the tcp stack, and ready to simply copy back to the user pages. Same for sendmsg, the socket is non-blocking, so I would not expect it to stall for unusually long periods. Maybe there is an issue hiding in how we call send/recv?
Quoted 3 lines
And the only way of inhibiting that would be to check _prior_ whether we can issue the call in the first place.
(That's why I did theSOCK_NOSPACEtests in the previous patchsets).
The socket is non-blocking. I always thought that SOCK_NOSPACE is serving blocking sockets... afaik, there shouldn't be a real difference between checking NOSPACE vs. attempting sendmsg on a non-blocking socket... But I could be wrong here..
Quoted 52 lines
Quoted 43 lines
Quoted 43 lines
drivers/nvme/host/tcp.c +2
-2} - return consumed; + return consumed - len; } static void nvme_tcp_data_ready(struct sock *sk)@@ -1258,7 +1269,7 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) return ret; } -static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue) +static int nvme_tcp_try_recv (struct nvme_tcp_queue *queue,unsigned long deadline) { struct socket *sock = queue->sock;struct sock *sk =
sock->sk;drivers/nvme/host/tcp.c +1
-0@@ -1269,6 +1280,7 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) rd_desc.count = 1; lock_sock(sk); queue->nr_cqe = 0; + queue->deadline = deadline; consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); release_sock(sk);return consumed;
drivers/nvme/host/tcp.c +0
-0@@ -1278,14 +1290,15 @@ static void nvme_tcp_io_work(struct work_struct *w) {struct
nvme_tcp_queue*queue =
container_of(w, struct nvme_tcp_queue, io_work); - unsigned long deadline = jiffies +msecs_to_jiffies(1); + unsigned longtx_deadline= jiffies +msecs_to_jiffies(1); + unsigned longrx_deadline=tx_deadline+msecs_to_jiffies(1), overrun;
do {
bool pending = false;
int result;
if (mutex_trylock(&queue->send_mutex)) {
- result =nvme_tcp_try_send(queue);
+ result =nvme_tcp_send_all(queue, tx_deadline);mutex_unlock(&queue->send_mutex);
if (result > 0)
pending = true;Quoted 9 lines
drivers/nvme/host/tcp.c static void nvme_tcp_io_work(struct · line 1309
Quoted 6 lines
drivers/nvme/host/tcp.c +1
-1@@ -1293,7 +1306,7 @@ static void nvme_tcp_io_work(struct work_struct *w) break; } - result = nvme_tcp_try_recv(queue); + result = nvme_tcp_try_recv(queue, rx_deadline);I think you want a more frequent substitution of sends/receives. the granularity of 1ms budget may be too coarse?
Quoted 2 lines
The problem is not the granularity, the problem is the latency spikes I'm seeing when issuing 'sendmsg' or '
read_sock'.
I think that its possible that there is another underlying issue. Because these calls should NOT be blocked for a long period of time for non-blocking sockets.
Quoted 22 lines
Quoted 13 lines
Quoted 13 lines
drivers/nvme/host/tcp.c +1
-1if (result > 0) pending = true; else if (unlikely(result < 0))@@ -1302,7 +1315,13 @@ static void nvme_tcp_io_work(struct work_struct *w) if (!pending || !queue->rd_enabled) return; - } while (!time_after(jiffies, deadline)); /* quota is exhausted */ + } while (!time_after(jiffies, rx_deadline)); /* quota isexhausted */
+
+ overrun = jiffies -rx_deadline;Quoted 9 lines
Quoted 5 lines
+ if (
nvme_tcp_queue_id(queue)> 0 &&
+ overrun >msecs_to_jiffies(10))
+ dev_dbg(queue->ctrl->ctrl.device, "queue %d: queue stall (%u msecs)\n",
+nvme_tcp_queue_id(queue),jiffies_to_msecs(overrun));Umm, ok. why 10? why not 2? or 3?
Do you expectio_workto spend more time executing?
Quoted 1 line
Eg 4000 msecs like on my testbed?
If you see ANYTHING in the IO path stalling for 4 seconds then we have a bigger issue here.
Quoted 32 lines
Yes.
Quoted 12 lines
Quoted 12 lines
queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); }drivers/nvme/host/tcp.c +1
-0@@ -2666,6 +2685,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { struct nvme_tcp_queue *queue = hctx->driver_data; struct sock *sk = queue->sock->sk; + unsigned long deadline = jiffies + msecs_to_jiffies(1); if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) return 0;@@ -2673,7 +2693,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)Quoted 8 lines
drivers/nvme/host/tcp.c static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) · line 2696
Quoted 5 lines
drivers/nvme/host/tcp.c +1
-1set_bit(NVME_TCP_Q_POLLING, &queue->flags); if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) sk_busy_loop(sk, true); - nvme_tcp_try_recv(queue); + nvme_tcp_try_recv(queue, deadline);spend a millisecond in
nvme_tcp_poll()??
Isn't it too long?Haven't tried with polling, so can't honestly answer.
In the end, it all boils down to numbers.
I'm having 2 controllers with 32 queues and 256 requests. Running on a 10GigE link one should be getting a net throughput of 1GB/s, or, with 4k requests, 256k IOPS.
Or a latency of 0.25 milliseconds per command.
In the optimal case. As I'm seeing a bandwidth of around 500MB/s, I'm looking at a latency 0.5 milliseconds_inthe idealcase_.
Quoted 1 line
So no, I don't think the 1 milliseconds is too long.
nvme_tcp_poll is NOT designed to poll for the entirety or an individual IO lifetime. Its a non-selective polling interface wired to io_uring. It should peek the socket, consume what ever is there, and break. It most definitely should not spend 1ms in this check...
Re: [PATCH 1/3] nvme-tcp: improve rx/tx fairness
Quoted 6 lines · Sagi Grimberg
Quoted 3 lines · Hannes Reinecke
Quoted 2 lines · Sagi Grimberg
Hey Hannes, thanks for doing this.
[ .. ]
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
That is a significant decline in relative perf. I'm counting 12.5%... Can you explain why that is?
Not really :-(
But then fairness cuts both ways; so I am not surprised that some workloads suffer here.Well, 12.5% is rather steep, don't you agree?
I'm even more surprised that seq/rand read make a difference...
Yeah; but I guess I found the issue now (we should interrupt data transfers if the deadline expires, not just pdu transfers).
[ .. ]
Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
I don't see why you need to keep this in the queue struct. You could have easily initialize it in the
read_descriptor_tand test against it.Because I wanted to interrupt the receive side for large data transfers, and haven't found a way to pass the deadline into ->read_sock().
desc.count is for you to interpret however you want it to be no?
Except for the magic 'desc.count == 0' value; but yeah, that should work.
Quoted 53 lines
Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
/* send state */
structnvme_tcp_request*request;drivers/nvme/host/tcp.c +2
-1@@ -359,14 +360,18 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, } } -static inline void nvme_tcp_send_all (struct nvme_tcp_queue *queue) +static inline int nvme_tcp_send_all (struct nvme_tcp_queue *queue, + unsigned long deadline) { int ret; /* drain the send queue as much as we can... */ do {Quoted 9 lines
drivers/nvme/host/tcp.c static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, · line 374
Quoted 5 lines
drivers/nvme/host/tcp.c +3
-0ret = nvme_tcp_try_send(queue); + if (time_after(jiffies, deadline)) + break; } while (ret > 0); + return ret;I think you want a different interface,
nvme_tcp_send_budgeted(queue, budget). I don't know what you pass here, but jiffies is a rather large granularity...Quoted 3 lines
Hmm. I've been using jiffies as the
io_workloop had been counting in jiffies. But let me check what would happen if I move that over to PDU / size counting.Cool.
Quoted 24 lines
Quoted 24 lines
Quoted 14 lines
drivers/nvme/host/tcp.c +1
-0} static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue)@@ -385,6 +390,7 @@ static inline void nvme_tcp_queue_request(structnvme_tcp_request *req, bool sync, bool last) { struct nvme_tcp_queue *queue = req->queue; + unsigned long deadline = jiffies + msecs_to_jiffies(1); bool empty;empty =
llist_add(&req->lentry, &queue->req_list)&&(quoted diff) +0
-0@@ -397,7 +403,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, */Quoted 6 lines
if (
queue->io_cpu==raw_smp_processor_id()&& sync && empty &&mutex_trylock(&queue->send_mutex)) { -nvme_tcp_send_all(queue);
+nvme_tcp_send_all(queue, deadline);mutex_unlock(&queue->send_mutex);
}Umm, spend up to a millisecond in in
queue_request? Sounds like way too much... Did you ever see this deadline exceeded? sends should be rather quick...
Quoted 6 lines
Quoted 3 lines
Of _course_ it's too long. That's kinda the point.
But I'm seeing network latency up to 4000 msecs (!) on my test setup, so _that_ is the least of my worries ...This has nothing to do with the network latency afaict. This is just a matter of how long does it take to write to the socket buffer (or in our case, construct skbs and add them to the socket as TX is zero copy).
Not the transfer itself? I would have thought so ...
Quoted 35 lines
Quoted 15 lines
Quoted 15 lines
drivers/nvme/host/tcp.c static int nvme_tcp_recv_skb(read_descriptor_t · line 973
Quoted 11 lines
drivers/nvme/host/tcp.c +5
-0@@ -959,9 +965,14 @@ static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,nvme_tcp_error_recovery (&queue->ctrl->ctrl); return result; } + if (time_after(jiffies, queue->deadline)) { + desc->count = 0; + break; + } +That is still not right.
You don't want to spend a full deadline reading from the socket, and then spend a full deadline writing to the socket...Quoted 6 lines
Yes, and no.
Problem is that the current code serializes writes and reads. Essentially for each iteration we first to a write, and then a read. If we spend the full deadline on write we will need to reschedule, but we then _again_ start with writes. This leads to a heavy preference for writing, and negative performance impact.Yes, the assumption we should take is that the tx/rx limits allow to switch between the two at least once or twice in the
io_workwhile loop, and that they get a fair'ish share of quota.Quoted 8 lines
Quoted 5 lines
You want the
io_workto take a full deadline, and send budgets oftry_sendandtry_recv. And set it to sane counts. Say 8 pdus, or 64k bytes. We want to get to some magic value that presents a
sane behavior, that confidently fits inside a deadline, and is fair.Easier said than done.
Biggest problem is that most of the latency increase comes from the actual 'sendmsg()' and 'read_sock()' calls.
Quoted 6 lines
This is a bit surprising. because
read_sockreads data that is already processed by the tcp stack, and ready to simply copy back to the user pages. Same for sendmsg, the socket is non-blocking, so I would not expect it to stall for unusually long periods. Maybe there is an issue hiding in how we call send/recv?
Hmm. That could be.
Quoted 8 lines
Quoted 3 lines
And the only way of inhibiting that would be to check _prior_ whether we can issue the call in the first place.
(That's why I did theSOCK_NOSPACEtests in the previous patchsets).The socket is non-blocking. I always thought that
SOCK_NOSPACEis serving blocking sockets... afaik, there shouldn't be a real difference between checking NOSPACE vs. attempting sendmsg on a non-blocking socket... But I could be wrong here..
Weellll ... if 'SOCK_NOSPACE' is for blocking sockets, why do we even get the 'write_space()' callback?
It gets triggered quite often, and checking for the SOCK_NOSPACE bit before sending drops the number of invocations quite significantly.
[ .. ]
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
drivers/nvme/host/tcp.c +1
-1break; } - result = nvme_tcp_try_recv(queue); + result = nvme_tcp_try_recv(queue, rx_deadline);
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
I think you want a more frequent substitution of sends/receives. the granularity of 1ms budget may be too coarse?
The problem is not the granularity, the problem is the latency spikes I'm seeing when issuing 'sendmsg' or '
read_sock'.I think that its possible that there is another underlying issue. Because these calls should NOT be blocked for a long period of time for non-blocking sockets.
I did some instrumentation, and that certainly pointed into that direction. But will have to redo that to get some meaningful data here.
Another point really might be the granularity; jiffies isn't a good measurement here (even with HZ=1000 we'll only have milliseconds granularity, so it might be that we're incurring fluctuations due to jiffy calculation itself).
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 10 lines
(quoted diff) +0
-0if (result > 0) pending = true;else if (unlikely(result < 0))
drivers/nvme/host/tcp.c +1
-1@@ -1302,7 +1315,13 @@ static void nvme_tcp_io_work(struct work_struct *w) if (!pending || !queue->rd_enabled) return; - } while (!time_after(jiffies, deadline)); /* quota is exhausted */ + } while (!time_after(jiffies, rx_deadline)); /* quota isQuoted 8 lines
exhausted */
+
+ overrun = jiffies -rx_deadline;
+ if (nvme_tcp_queue_id(queue)> 0 &&
+ overrun >msecs_to_jiffies(10))
+ dev_dbg(queue->ctrl->ctrl.device, "queue %d: queue stall (%u msecs)\n",
+nvme_tcp_queue_id(queue),jiffies_to_msecs(overrun));Umm, ok. why 10? why not 2? or 3?
Do you expectio_workto spend more time executing?
Quoted 4 lines
Quoted 1 line
Eg 4000 msecs like on my testbed?
If you see ANYTHING in the IO path stalling for 4 seconds then we have a bigger issue here.
You don't say ...
Quoted 32 lines
Quoted 32 lines
Yes.
Quoted 12 lines
Quoted 12 lines
queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); }drivers/nvme/host/tcp.c +1
-0@@ -2666,6 +2685,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { struct nvme_tcp_queue *queue = hctx->driver_data; struct sock *sk = queue->sock->sk; + unsigned long deadline = jiffies + msecs_to_jiffies(1); if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) return 0;@@ -2673,7 +2693,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)Quoted 8 lines
drivers/nvme/host/tcp.c static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) · line 2696
Quoted 5 lines
drivers/nvme/host/tcp.c +1
-1set_bit(NVME_TCP_Q_POLLING, &queue->flags); if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) sk_busy_loop(sk, true); - nvme_tcp_try_recv(queue); + nvme_tcp_try_recv(queue, deadline);spend a millisecond in
nvme_tcp_poll()??
Isn't it too long?Haven't tried with polling, so can't honestly answer.
In the end, it all boils down to numbers.
I'm having 2 controllers with 32 queues and 256 requests. Running on a 10GigE link one should be getting a net throughput of 1GB/s, or, with 4k requests, 256k IOPS.
Or a latency of 0.25 milliseconds per command.
In the optimal case. As I'm seeing a bandwidth of around 500MB/s, I'm looking at a latency 0.5 milliseconds_inthe idealcase_.
Quoted 5 lines
Quoted 1 line
So no, I don't think the 1 milliseconds is too long.
nvme_tcp_pollis NOT designed to poll for the entirety or an individual IO lifetime. Its a non-selective polling interface wired toio_uring. It should peek the socket, consume what ever is there, and break. It most definitely should not spend 1ms in this check...
Oh, I know that.
That's why I said: "Haven't tried with polling."
The calculation is independent on polling.
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Sagi Grimberg
maintainerRe: [PATCH 1/3] nvme-tcp: improve rx/tx fairness
Quoted 84 lines · Hannes Reinecke
Quoted 6 lines · Sagi Grimberg
Quoted 3 lines · Hannes Reinecke
Quoted 2 lines · Sagi Grimberg
Hey Hannes, thanks for doing this.
[ .. ]
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
That is a significant decline in relative perf. I'm counting 12.5%... Can you explain why that is?
Not really :-(
But then fairness cuts both ways; so I am not surprised that some workloads suffer here.Well, 12.5% is rather steep, don't you agree?
I'm even more surprised that seq/rand read make a difference...Yeah; but I guess I found the issue now (we should interrupt data transfers if the deadline expires, not just pdu transfers).
[ .. ]
Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
I don't see why you need to keep this in the queue struct. You could have easily initialize it in the
read_descriptor_tand test against it.Because I wanted to interrupt the receive side for large data transfers, and haven't found a way to pass the deadline into ->read_sock().
desc.count is for you to interpret however you want it to be no?
Except for the magic 'desc.count == 0' value; but yeah, that should work.
Quoted 53 lines
Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
/* send state */
structnvme_tcp_request*request;drivers/nvme/host/tcp.c +2
-1@@ -359,14 +360,18 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, } } -static inline void nvme_tcp_send_all (struct nvme_tcp_queue *queue) +static inline int nvme_tcp_send_all (struct nvme_tcp_queue *queue, + unsigned long deadline) { int ret; /* drain the send queue as much as we can... */ do {Quoted 9 lines
drivers/nvme/host/tcp.c static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, · line 374
Quoted 5 lines
drivers/nvme/host/tcp.c +3
-0ret = nvme_tcp_try_send(queue); + if (time_after(jiffies, deadline)) + break; } while (ret > 0); + return ret;I think you want a different interface,
nvme_tcp_send_budgeted(queue, budget). I don't know what you pass here, but jiffies is a rather large granularity...Quoted 3 lines
Hmm. I've been using jiffies as the
io_workloop had been counting in jiffies. But let me check what would happen if I move that over to PDU / size counting.Cool.
Quoted 24 lines
Quoted 24 lines
Quoted 14 lines
drivers/nvme/host/tcp.c +1
-0} static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue)@@ -385,6 +390,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, bool sync, bool last) { struct nvme_tcp_queue *queue = req->queue; + unsigned long deadline = jiffies + msecs_to_jiffies(1); bool empty;empty =
llist_add(&req->lentry, &queue->req_list)&&(quoted diff) +0
-0@@ -397,7 +403,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, */Quoted 6 lines
if (
queue->io_cpu==raw_smp_processor_id()&& sync && empty &&mutex_trylock(&queue->send_mutex)) { -nvme_tcp_send_all(queue);
+nvme_tcp_send_all(queue, deadline);mutex_unlock(&queue->send_mutex);
}Umm, spend up to a millisecond in in
queue_request? Sounds like way too much... Did you ever see this deadline exceeded? sends should be rather quick...
Quoted 7 lines
Quoted 6 lines
Quoted 3 lines
Of _course_ it's too long. That's kinda the point.
But I'm seeing network latency up to 4000 msecs (!) on my test setup, so _that_ is the least of my worries ...This has nothing to do with the network latency afaict. This is just a matter of how long does it take to write to the socket buffer (or in our case, construct skbs and add them to the socket as TX is zero copy).
Not the transfer itself? I would have thought so ...
It may trigger push packets down to [tcp/ip]_output under some conditions, but that is also not something that should block for long.
Quoted 48 lines
Quoted 36 lines
Quoted 16 lines
Quoted 16 lines
drivers/nvme/host/tcp.c static int · line 973
Quoted 12 lines
drivers/nvme/host/tcp.c +5
-0@@ -959,9 +965,14 @@ static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, nvme_tcp_error_recovery(&queue->ctrl->ctrl); return result; } + if (time_after(jiffies, queue->deadline)) { + desc->count = 0; + break; + } +That is still not right.
You don't want to spend a full deadline reading from the socket, and then spend a full deadline writing to the socket...Quoted 6 lines
Yes, and no.
Problem is that the current code serializes writes and reads. Essentially for each iteration we first to a write, and then a read. If we spend the full deadline on write we will need to reschedule, but we then _again_ start with writes. This leads to a heavy preference for writing, and negative performance impact.Yes, the assumption we should take is that the tx/rx limits allow to switch between the two at least once or twice in the
io_workwhile loop, and that they get a fair'ish share of quota.Quoted 8 lines
Quoted 5 lines
You want the
io_workto take a full deadline, and send budgets oftry_sendandtry_recv. And set it to sane counts. Say 8 pdus, or 64k bytes. We want to get to some magic value that presents a
sane behavior, that confidently fits inside a deadline, and is fair.Easier said than done.
Biggest problem is that most of the latency increase comes from the actual 'sendmsg()' and 'read_sock()' calls.Quoted 6 lines
This is a bit surprising. because
read_sockreads data that is already processed by the tcp stack, and ready to simply copy back to the user pages. Same for sendmsg, the socket is non-blocking, so I would not expect it to stall for unusually long periods. Maybe there is an issue hiding in how we call send/recv?Hmm. That could be.
Quoted 4 lines
Quoted 3 lines
And the only way of inhibiting that would be to check _prior_ whether we can issue the call in the first place.
(That's why I did theSOCK_NOSPACEtests in the previous patchsets).
Quoted 7 lines
Quoted 4 lines
The socket is non-blocking. I always thought that
SOCK_NOSPACEis serving blocking sockets... afaik, there shouldn't be a real difference between checking NOSPACE vs. attempting sendmsg on a non-blocking socket... But I could be wrong here..Weellll ... if '
SOCK_NOSPACE' is for blocking sockets, why do we even get the 'write_space()' callback?
It gets triggered quite often, and checking for theSOCK_NOSPACEbit before sending drops the number of invocations quite significantly.
I need to check, but where have you testing it? in the inline path? Thinking further, I do agree that because the io_work is shared, we may sendmsg immediately as space is becomes available instead of some minimum space.
Can you please quantify this with your testing?
How many times we get write_space() callback? how many times we get EAGAIN, and what is the perf... Also interesting if this is more apparent in specific workloads. I'm assuming its more apparent with large write workloads.
Re: [PATCH 1/3] nvme-tcp: improve rx/tx fairness
Quoted 3 lines · Sagi Grimberg
On 08/07/2024 18:50, Hannes Reinecke wrote:
[ .. ]
Quoted 11 lines
Quoted 3 lines
Weellll ... if '
SOCK_NOSPACE' is for blocking sockets, why do we even get the 'write_space()' callback?
It gets triggered quite often, and checking for theSOCK_NOSPACEbit before sending drops the number of invocations quite significantly.I need to check, but where have you testing it? in the inline path? Thinking further, I do agree that because the
io_workis shared, we may sendmsg immediately as space is becomes available instead of some minimum space.Quoted 1 line
Can you please quantify this with your testing?
How many times we get
write_space()callback? how many times we get EAGAIN, and what is the perf... Also interesting if this is more apparent in specific workloads. I'm assuming its more apparent with large write workloads.
Stats for my testing
(second row is controller 1, third row controller 2):
write_space:
0: 0 0
1: 489 2
10: 325 20
11: 28 1
12: 14 1
13: 252 1
14: 100 12
15: 310 1
16: 454 2
2: 31 11
3: 50 1
4: 299 42
5: 1 19
6: 12 0
7: 737 16
8: 636 1
9: 19 1
queue_busy:
0: 0 0
1: 396 2
10: 66 18
11: 91 8
12: 56 14
13: 464 7
14: 24 15
15: 574 8
16: 516 9
2: 22 29
3: 56 5
4: 153 99
5: 2 40
6: 18 0
7: 632 5
8: 590 13
9: 129 2
The send latency is actually pretty good, and around 150 us on controller 1 (with two notable exceptions of 480 us on queue 5 and 646 us on queue 6) and around 100 us on controller 2 (again with exceptions of 444 us on queue 5 and 643 us on queue 6). Each queue processing around 150k requests.
Receive latency is far more consistent; each queue has around 150 us on controller 1 and 140 us on controller 2.
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Sagi Grimberg
maintainerRe: [PATCH 1/3] nvme-tcp: improve rx/tx fairness
Quoted 17 lines · Hannes Reinecke
Quoted 3 lines · Sagi Grimberg
On 08/07/2024 18:50, Hannes Reinecke wrote:
[ .. ]
Quoted 11 lines
Quoted 3 lines
Weellll ... if '
SOCK_NOSPACE' is for blocking sockets, why do we even get the 'write_space()' callback?
It gets triggered quite often, and checking for theSOCK_NOSPACEbit before sending drops the number of invocations quite significantly.I need to check, but where have you testing it? in the inline path?
Thinking further, I do agree that because theio_workis shared, we may
sendmsg immediately as space is becomes available instead of some minimum space. > Can you please quantify this with your testing?
How many times we getwrite_space()callback? how many times we get EAGAIN, and what is the perf... Also interesting if this is more apparent in specific workloads. I'm assuming its more apparent with large write workloads.
Quoted 2 lines
Stats for my testing
(second row is controller 1, third row controller 2):
Which row? you mean column?
Quoted 11 lines
write_space:
0: 0 0
1: 489 2
10: 325 20
11: 28 1
12: 14 1
13: 252 1
14: 100 12
15: 310 1
16: 454 2
Quoted 8 lines
2: 31 11
3: 50 1
4: 299 42
5: 1 19
6: 12 0
7: 737 16
8: 636 1
9: 19 1
These are incrementing counters where ctrl1 and ctrl2 got the .write_space() callbacks? What are 0-16? queues? I thought you had 32 queues...
Quoted 11 lines
queue_busy:
0: 0 0
1: 396 2
10: 66 18
11: 91 8
12: 56 14
13: 464 7
14: 24 15
15: 574 8
16: 516 9
Quoted 8 lines
2: 22 29
3: 56 5
4: 153 99
5: 2 40
6: 18 0
7: 632 5
8: 590 13
9: 129 2
What is queue_busy?
Quoted 2 lines
The send latency is actually pretty good, and around 150 us on controller 1
150us for what? the average duration of sendmsg call alone?
i.e.
start
sendmsg
end
Quoted 1 line
(with two notable exceptions of 480 us on queue 5 and 646 us on queue 6) and around 100 us on controller 2 (again with exceptions of 444 us on queue 5 and 643 us on queue 6). Each queue processing around 150k requests.
Are you referring to the above measurements? I'm having trouble understanding what you wrote...
Quoted 2 lines
Receive latency is far more consistent; each queue has around 150 us on controller 1 and 140 us on controller 2.
Again, latency of what?
PATCH v2 2/3 nvme-tcp: align I/O cpu with blk-mq mapping
In reply to Hannes ReineckeWe should align the 'io_cpu' setting with the blk-mq cpu mapping to ensure that we're not bouncing threads when doing I/O. To avoid cpu contention this patch also adds an atomic counter for the number of queues on each cpu to distribute the load across all CPUs in the blk-mq cpu set. Additionally we should always set the 'io_cpu' value, as in the WQ_UNBOUND case it'll be treated as a hint anyway.
Performance comparison:
baseline rx/tx blk-mq align
4k seq write: 449MiB/s 480MiB/s 524MiB/s
4k rand write: 410MiB/s 481MiB/s 524MiB/s
4k seq read: 478MiB/s 481MiB/s 566MiB/s
4k rand read: 547MiB/s 480MiB/s 511MiB/s
--- drivers/nvme/host/tcp.c | 65 +++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 16 deletions(-)
drivers/nvme/host/tcp.c +49 -16
@@ -26,6 +26,8 @@struct nvme_tcp_queue ; +static atomic_t nvme_tcp_cpu_queues[NR_CPUS ]; +/* Define the socket priority to use for connections were it is desirable * that the NIC consider performing optimized packet processing or filtering. * A non-zero value being sufficient to indicate general consideration of any
@@ -1578,20 +1580,42 @@ static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)static void nvme_tcp_set_queue_io_cpu (struct nvme_tcp_queue *queue) {struct nvme_tcp_ctrl *ctrl = queue->ctrl; -int qid =nvme_tcp_queue_id (queue); -int n =0 ; - -if (nvme_tcp_default_queue (queue)) - n = qid -1 ; -else if (nvme_tcp_read_queue (queue)) - n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT ] -1 ; -else if (nvme_tcp_poll_queue (queue)) +struct blk_mq_tag_set *set = &ctrl->tag_set; +int qid =nvme_tcp_queue_id (queue) -1 ; +unsigned int *mq_map =NULL ;; +int n =0 , cpu, io_cpu, min_queues =WORK_CPU_UNBOUND ; + +if (nvme_tcp_default_queue (queue)) { + mq_map = set->map[HCTX_TYPE_DEFAULT ].mq_map; + n = qid; + }else if (nvme_tcp_read_queue (queue)) { + mq_map = set->map[HCTX_TYPE_READ ].mq_map; + n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT ]; + }else if (nvme_tcp_poll_queue (queue)) { + mq_map = set->map[HCTX_TYPE_POLL ].mq_map; n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT ] - - ctrl->io_queues[HCTX_TYPE_READ ] -1 ; -if (wq_unbound) - queue->io_cpu =WORK_CPU_UNBOUND ; -else - queue->io_cpu =cpumask_next_wrap (n -1 , cpu_online_mask,-1 , false); + ctrl->io_queues[HCTX_TYPE_READ ]; + } + +if (WARN_ON (!mq_map)) +return ; +for_each_online_cpu (cpu) { +int num_queues; + +if (mq_map[cpu] != qid) +continue ; + num_queues =atomic_read (&nvme_tcp_cpu_queues[cpu]); +if (num_queues < min_queues) { + min_queues = num_queues; + io_cpu = cpu; + } + } +if (io_cpu != queue->io_cpu) { + queue->io_cpu = io_cpu; +atomic_inc (&nvme_tcp_cpu_queues[io_cpu]); + } +dev_dbg (ctrl->ctrl.device,"queue %d: using cpu %d\n" , + qid, queue->io_cpu); }static void nvme_tcp_tls_done (void *data,int status,key_serial_t pskid)
@@ -1735,7 +1759,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, queue->sock->sk->sk_allocation =GFP_ATOMIC ; queue->sock->sk->sk_use_task_frag = false; -nvme_tcp_set_queue_io_cpu (queue); + queue->io_cpu =WORK_CPU_UNBOUND ; queue->request =NULL ; queue->data_remaining =0 ; queue->ddgst_remaining =0 ;
@@ -1847,6 +1871,10 @@ static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)kernel_sock_shutdown (queue->sock,SHUT_RDWR );nvme_tcp_restore_sock_ops (queue);cancel_work_sync (&queue->io_work); +if (queue->io_cpu !=WORK_CPU_UNBOUND ) { +atomic_dec (&nvme_tcp_cpu_queues[queue->io_cpu]); + queue->io_cpu =WORK_CPU_UNBOUND ; + } }static void nvme_tcp_stop_queue (struct nvme_ctrl *nctrl,int qid)
@@ -1891,9 +1919,10 @@ static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)nvme_tcp_init_recv_ctx (queue);nvme_tcp_setup_sock_ops (queue); -if (idx) +if (idx) { +nvme_tcp_set_queue_io_cpu (queue); ret =nvmf_connect_io_queue (nctrl, idx); -else + }else ret =nvmf_connect_admin_queue (nctrl);if (!ret) {
@@ -2920,6 +2949,7 @@ static struct nvmf_transport_ops nvme_tcp_transport = {static int __init nvme_tcp_init_module (void ) {unsigned int wq_flags =WQ_MEM_RECLAIM |WQ_HIGHPRI |WQ_SYSFS ; +int cpu;BUILD_BUG_ON (sizeof (struct nvme_tcp_hdr ) !=8 );BUILD_BUG_ON (sizeof (struct nvme_tcp_cmd_pdu ) !=72 );
@@ -2937,6 +2967,9 @@ static int __init nvme_tcp_init_module(void)if (!nvme_tcp_wq)return -ENOMEM ; +for_each_possible_cpu (cpu) +atomic_set (&nvme_tcp_cpu_queues[cpu],0 ); +nvmf_register_transport (&nvme_tcp_transport);return 0 ; }
Signature · 1 line
2.35.3
Sagi Grimberg
maintainerRe: [PATCH 2/3] nvme-tcp: align I/O cpu with blk-mq mapping
Quoted 37 lines · Hannes Reinecke
We should align the '
io_cpu' setting with the blk-mq cpu mapping to ensure that we're not bouncing threads when doing I/O. To avoid cpu contention this patch also adds an atomic counter for the number of queues on each cpu to distribute the load across all CPUs in the blk-mq cpu set. Additionally we should always set the 'io_cpu' value, as in theWQ_UNBOUNDcase it'll be treated as a hint anyway.Performance comparison:
baseline rx/tx blk-mq align
4k seq write: 449MiB/s 480MiB/s 524MiB/s
4k rand write: 410MiB/s 481MiB/s 524MiB/s
4k seq read: 478MiB/s 481MiB/s 566MiB/s
4k rand read: 547MiB/s 480MiB/s 511MiB/sSigned-off-by Hannes Reinecke <[email protected]>--- drivers/nvme/host/tcp.c | 65 +++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 16 deletions(-)drivers/nvme/host/tcp.c +2
-0@@ -26,6 +26,8 @@struct nvme_tcp_queue ; +static atomic_t nvme_tcp_cpu_queues[NR_CPUS ]; +/* Define the socket priority to use for connections were it is desirable * that the NIC consider performing optimized packet processing or filtering. * A non-zero value being sufficient to indicate general consideration of any @@ -1578,20 +1580,42 @@ static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)static void nvme_tcp_set_queue_io_cpu (struct nvme_tcp_queue *queue) {struct nvme_tcp_ctrl *ctrl = queue->ctrl;
drivers/nvme/host/tcp.c static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) · line 1584
Quoted 10 lines
drivers/nvme/host/tcp.c +2
-8-int qid =nvme_tcp_queue_id (queue); -int n =0 ; - -if (nvme_tcp_default_queue (queue)) - n = qid -1 ; -else if (nvme_tcp_read_queue (queue)) - n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT ] -1 ; -else if (nvme_tcp_poll_queue (queue)) +struct blk_mq_tag_set *set = &ctrl->tag_set; +int qid =nvme_tcp_queue_id (queue) -1 ;
umm, its not the qid, why change it? I mean it looks harmless, but I don't see why.
drivers/nvme/host/tcp.c static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) · line 1586
Quoted 2 lines
drivers/nvme/host/tcp.c +2
-0+unsigned int *mq_map =NULL ;; +int n =0 , cpu, io_cpu, min_queues =WORK_CPU_UNBOUND ;
min_queues initialization looks very very weird. I can't even parse it. besides, why is it declared in this scope?
Quoted 25 lines
drivers/nvme/host/tcp.c +19
-5+ +if (nvme_tcp_default_queue (queue)) { + mq_map = set->map[HCTX_TYPE_DEFAULT ].mq_map; + n = qid; + }else if (nvme_tcp_read_queue (queue)) { + mq_map = set->map[HCTX_TYPE_READ ].mq_map; + n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT ]; + }else if (nvme_tcp_poll_queue (queue)) { + mq_map = set->map[HCTX_TYPE_POLL ].mq_map; n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT ] - - ctrl->io_queues[HCTX_TYPE_READ ] -1 ; -if (wq_unbound) - queue->io_cpu =WORK_CPU_UNBOUND ; -else - queue->io_cpu =cpumask_next_wrap (n -1 , cpu_online_mask,-1 , false); + ctrl->io_queues[HCTX_TYPE_READ ]; + } + +if (WARN_ON (!mq_map)) +return ; +for_each_online_cpu (cpu) { +int num_queues; + +if (mq_map[cpu] != qid) +continue ;
drivers/nvme/host/tcp.c static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) · line 1616
Quoted 10 lines
drivers/nvme/host/tcp.c +10
-0+ num_queues =atomic_read (&nvme_tcp_cpu_queues[cpu]); +if (num_queues < min_queues) { + min_queues = num_queues; + io_cpu = cpu; + } + } +if (io_cpu != queue->io_cpu) { + queue->io_cpu = io_cpu; +atomic_inc (&nvme_tcp_cpu_queues[io_cpu]); + }
Why is that conditioned ? why not always assign and inc the counter?
Quoted 15 lines
drivers/nvme/host/tcp.c +3
-1+dev_dbg (ctrl->ctrl.device,"queue %d: using cpu %d\n" , + qid, queue->io_cpu); }static void nvme_tcp_tls_done (void *data,int status,key_serial_t pskid)@@ -1735,7 +1759,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, queue->sock->sk->sk_allocation =GFP_ATOMIC ; queue->sock->sk->sk_use_task_frag = false; -nvme_tcp_set_queue_io_cpu (queue); + queue->io_cpu =WORK_CPU_UNBOUND ; queue->request =NULL ; queue->data_remaining =0 ; queue->ddgst_remaining =0 ;
drivers/nvme/host/tcp.c static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) · line 1877
Quoted 7 lines
drivers/nvme/host/tcp.c +4
-0@@ -1847,6 +1871,10 @@ static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)kernel_sock_shutdown (queue->sock,SHUT_RDWR );nvme_tcp_restore_sock_ops (queue);cancel_work_sync (&queue->io_work); +if (queue->io_cpu !=WORK_CPU_UNBOUND ) { +atomic_dec (&nvme_tcp_cpu_queues[queue->io_cpu]); + queue->io_cpu =WORK_CPU_UNBOUND ; + }
Does anything change if we still set the io_cpu in wq_unbound case? If not, I think we should always do it and simplify the code.
Quoted 25 lines
drivers/nvme/host/tcp.c +4
-2}static void nvme_tcp_stop_queue (struct nvme_ctrl *nctrl,int qid)@@ -1891,9 +1919,10 @@ static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)nvme_tcp_init_recv_ctx (queue);nvme_tcp_setup_sock_ops (queue); -if (idx) +if (idx) { +nvme_tcp_set_queue_io_cpu (queue); ret =nvmf_connect_io_queue (nctrl, idx); -else + }else ret =nvmf_connect_admin_queue (nctrl);if (!ret) {@@ -2920,6 +2949,7 @@ static struct nvmf_transport_ops nvme_tcp_transport = {static int __init nvme_tcp_init_module (void ) {unsigned int wq_flags =WQ_MEM_RECLAIM |WQ_HIGHPRI |WQ_SYSFS ; +int cpu;BUILD_BUG_ON (sizeof (struct nvme_tcp_hdr ) !=8 );BUILD_BUG_ON (sizeof (struct nvme_tcp_cmd_pdu ) !=72 );
drivers/nvme/host/tcp.c static int __init nvme_tcp_init_module(void) · line 2972
Quoted 6 lines
drivers/nvme/host/tcp.c +3
-0@@ -2937,6 +2967,9 @@ static int __init nvme_tcp_init_module(void)if (!nvme_tcp_wq)return -ENOMEM ; +for_each_possible_cpu (cpu) +atomic_set (&nvme_tcp_cpu_queues[cpu],0 ); +
Why?
Re: [PATCH 2/3] nvme-tcp: align I/O cpu with blk-mq mapping
Quoted 53 lines · Sagi Grimberg
Quoted 36 lines · Hannes Reinecke
We should align the '
io_cpu' setting with the blk-mq cpu mapping to ensure that we're not bouncing threads when doing I/O. To avoid cpu contention this patch also adds an atomic counter for the number of queues on each cpu to distribute the load across all CPUs in the blk-mq cpu set. Additionally we should always set the 'io_cpu' value, as in theWQ_UNBOUNDcase it'll be treated as a hint anyway.Performance comparison:
baseline rx/tx blk-mq align
4k seq write: 449MiB/s 480MiB/s 524MiB/s
4k rand write: 410MiB/s 481MiB/s 524MiB/s
4k seq read: 478MiB/s 481MiB/s 566MiB/s
4k rand read: 547MiB/s 480MiB/s 511MiB/sSigned-off-by Hannes Reinecke <[email protected]>---
drivers/nvme/host/tcp.c| 65 +++++++++++++++++++++++++++++++----------1 file changed, 49 insertions(+), 16 deletions(-)drivers/nvme/host/tcp.c +2
-0@@ -26,6 +26,8 @@ struct nvme_tcp_queue; +static atomic_t nvme_tcp_cpu_queues[NR_CPUS ]; + /* Define the socket priority to use for connections were it is desirable* that the NIC consider performing optimized packet processing or filtering. * A non-zero value being sufficient to indicate general consideration of any @@ -1578,20 +1580,42 @@ static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) { struct nvme_tcp_ctrl *ctrl = queue->ctrl;drivers/nvme/host/tcp.c static bool nvme_tcp_poll_queue(struct · line 1585
Quoted 10 lines
drivers/nvme/host/tcp.c +2
-8- int qid = nvme_tcp_queue_id(queue); - int n = 0; - - if (nvme_tcp_default_queue(queue)) - n = qid - 1; - else if (nvme_tcp_read_queue(queue)) - n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; - else if (nvme_tcp_poll_queue(queue)) + struct blk_mq_tag_set *set = &ctrl->tag_set; + int qid = nvme_tcp_queue_id(queue) - 1;umm, its not the qid, why change it? I mean it looks harmless, but I don't see why.
Quoted 6 lines
drivers/nvme/host/tcp.c static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) · line 1586
Quoted 2 lines
drivers/nvme/host/tcp.c +2
-0+ unsigned int *mq_map = NULL;; + int n = 0, cpu, io_cpu, min_queues = WORK_CPU_UNBOUND;
min_queuesinitialization looks very very weird. I can't even parse it. besides, why is it declared in this scope?
Because it's evaluated in the loop, with the value carried over across loop iterations.WORK_CPU_UNBOUND is the init value, to detect if it had been set at all.
Quoted 30 lines
Quoted 30 lines
drivers/nvme/host/tcp.c +9
-0+ + if (nvme_tcp_default_queue(queue)) { + mq_map = set->map[HCTX_TYPE_DEFAULT].mq_map; + n = qid; + } else if (nvme_tcp_read_queue(queue)) { + mq_map = set->map[HCTX_TYPE_READ].mq_map; + n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT]; + } else if (nvme_tcp_poll_queue(queue)) { + mq_map = set->map[HCTX_TYPE_POLL].mq_map;n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
- ctrl->io_queues[HCTX_TYPE_READ] - 1;
- if (wq_unbound)
-queue->io_cpu=WORK_CPU_UNBOUND;
- else
-queue->io_cpu=cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
+ ctrl->io_queues[HCTX_TYPE_READ];
+ }drivers/nvme/host/tcp.c +12
-0+ + if (WARN_ON(!mq_map)) + return; + for_each_online_cpu(cpu) { + int num_queues; + + if (mq_map[cpu] != qid) + continue; + num_queues = atomic_read(&nvme_tcp_cpu_queues[cpu]); + if (num_queues < min_queues) { + min_queues = num_queues; + io_cpu = cpu;
Quoted 9 lines
Quoted 6 lines
+ }
+ }
+ if (io_cpu!=queue->io_cpu) {
+queue->io_cpu=io_cpu;
+atomic_inc(&nvme_tcp_cpu_queues[io_cpu]); + }Why is that conditioned ? why not always assign and inc the counter?
Because we're having the conditional
if (mq_map[cpu] != qid)
above, so technically we might not find the correct mapping. Might be worthwhile doing a WARN_ON() here, but it might trigger with CPU hotplug...
Quoted 17 lines
Quoted 17 lines
+ dev_dbg(ctrl->ctrl.device, "queue %d: using cpu %d\n",
+ qid,queue->io_cpu);
}
static voidnvme_tcp_tls_done(void *data, int status, key_serial_t pskid)drivers/nvme/host/tcp.c +1
-1@@ -1735,7 +1759,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, queue->sock->sk->sk_allocation = GFP_ATOMIC; queue->sock->sk->sk_use_task_frag = false; - nvme_tcp_set_queue_io_cpu(queue); + queue->io_cpu = WORK_CPU_UNBOUND; queue->request = NULL; queue->data_remaining = 0; queue->ddgst_remaining = 0;@@ -1847,6 +1871,10 @@ static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) kernel_sock_shutdown(queue->sock, SHUT_RDWR); nvme_tcp_restore_sock_ops(queue);
Quoted 9 lines
drivers/nvme/host/tcp.c static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) · line 1877
Quoted 5 lines
drivers/nvme/host/tcp.c +4
-0cancel_work_sync(&queue->io_work); + if (queue->io_cpu != WORK_CPU_UNBOUND) { + atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]); + queue->io_cpu = WORK_CPU_UNBOUND; + }Does anything change if we still set the
io_cpuinwq_unboundcase? If not, I think we should always do it and simplify the code.
See above. WORK_CPU_UNBOUND means that we failed to find the cpu in the mq_map. Yes, I know, unlikely, but not impossible.
Quoted 22 lines
Quoted 22 lines
drivers/nvme/host/tcp.c +4
-2} static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)@@ -1891,9 +1919,10 @@ static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) nvme_tcp_init_recv_ctx(queue); nvme_tcp_setup_sock_ops(queue); - if (idx) + if (idx) { + nvme_tcp_set_queue_io_cpu(queue); ret = nvmf_connect_io_queue(nctrl, idx); - else + } else ret = nvmf_connect_admin_queue(nctrl); if (!ret) {@@ -2920,6 +2949,7 @@ static struct nvmf_transport_ops nvme_tcp_transport = { static int __init nvme_tcp_init_module(void) { unsigned int wq_flags = WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_SYSFS; + int cpu; BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8); BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72);
Quoted 8 lines
drivers/nvme/host/tcp.c static int __init nvme_tcp_init_module(void) · line 2971
Quoted 6 lines
drivers/nvme/host/tcp.c +3
-0@@ -2937,6 +2967,9 @@ static int __init nvme_tcp_init_module(void) if (!nvme_tcp_wq) return -ENOMEM; + for_each_possible_cpu(cpu) + atomic_set(&nvme_tcp_cpu_queues[cpu], 0); +Why?
Don't we need to initialize the atomic counters?
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Sagi Grimberg
maintainerRe: [PATCH 2/3] nvme-tcp: align I/O cpu with blk-mq mapping
Quoted 55 lines · Hannes Reinecke
Quoted 54 lines · Sagi Grimberg
Quoted 37 lines · Hannes Reinecke
We should align the '
io_cpu' setting with the blk-mq cpu mapping to ensure that we're not bouncing threads when doing I/O. To avoid cpu contention this patch also adds an atomic counter for the number of queues on each cpu to distribute the load across all CPUs in the blk-mq cpu set. Additionally we should always set the 'io_cpu' value, as in theWQ_UNBOUNDcase it'll be treated as a hint anyway.Performance comparison:
baseline rx/tx blk-mq align
4k seq write: 449MiB/s 480MiB/s 524MiB/s
4k rand write: 410MiB/s 481MiB/s 524MiB/s
4k seq read: 478MiB/s 481MiB/s 566MiB/s
4k rand read: 547MiB/s 480MiB/s 511MiB/sSigned-off-by Hannes Reinecke <[email protected]>---
drivers/nvme/host/tcp.c| 65
+++++++++++++++++++++++++++++++----------1 file changed, 49 insertions(+), 16 deletions(-)drivers/nvme/host/tcp.c +2
-0@@ -26,6 +26,8 @@ struct nvme_tcp_queue; +static atomic_t nvme_tcp_cpu_queues[NR_CPUS ]; + /* Define the socket priority to use for connections were it is desirable* that the NIC consider performing optimized packet processing or filtering. * A non-zero value being sufficient to indicate general consideration of any @@ -1578,20 +1580,42 @@ static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) { struct nvme_tcp_ctrl *ctrl = queue->ctrl;drivers/nvme/host/tcp.c static bool nvme_tcp_poll_queue(struct · line 1585
Quoted 10 lines
drivers/nvme/host/tcp.c +2
-8- int qid = nvme_tcp_queue_id(queue); - int n = 0; - - if (nvme_tcp_default_queue(queue)) - n = qid - 1; - else if (nvme_tcp_read_queue(queue)) - n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; - else if (nvme_tcp_poll_queue(queue)) + struct blk_mq_tag_set *set = &ctrl->tag_set; + int qid = nvme_tcp_queue_id(queue) - 1;umm, its not the qid, why change it? I mean it looks harmless, but I don't see why.
Quoted 8 lines
Quoted 6 lines
drivers/nvme/host/tcp.c static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) · line 1586
Quoted 2 lines
drivers/nvme/host/tcp.c +2
-0+ unsigned int *mq_map = NULL;; + int n = 0, cpu, io_cpu, min_queues = WORK_CPU_UNBOUND;
min_queuesinitialization looks very very weird. I can't even parse it. besides, why is it declared in this scope?Because it's evaluated in the loop, with the value carried over across loop iterations.
Right. missed that.
Quoted 1 line
WORK_CPU_UNBOUNDis the init value, to detect if it had been set at all.
as far as I read this patch, min_queues and num_queues are a quantity of queues. So it is bizzar to see it initiatlized to this completely unrelated value of WORK_CPU_UNBOUND. its really confusing. Still not sure what is tbh.
Quoted 40 lines
Quoted 31 lines
Quoted 31 lines
drivers/nvme/host/tcp.c +9
-0+ + if (nvme_tcp_default_queue(queue)) { + mq_map = set->map[HCTX_TYPE_DEFAULT].mq_map; + n = qid; + } else if (nvme_tcp_read_queue(queue)) { + mq_map = set->map[HCTX_TYPE_READ].mq_map; + n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT]; + } else if (nvme_tcp_poll_queue(queue)) { + mq_map = set->map[HCTX_TYPE_POLL].mq_map;n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - - ctrl->io_queues[HCTX_TYPE_READ] - 1;
- if (wq_unbound)
-queue->io_cpu=WORK_CPU_UNBOUND;
- else
-queue->io_cpu=cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
+ ctrl->io_queues[HCTX_TYPE_READ];
+ }drivers/nvme/host/tcp.c +12
-0+ + if (WARN_ON(!mq_map)) + return; + for_each_online_cpu(cpu) { + int num_queues; + + if (mq_map[cpu] != qid) + continue; + num_queues = atomic_read(&nvme_tcp_cpu_queues[cpu]); + if (num_queues < min_queues) { + min_queues = num_queues; + io_cpu = cpu;Quoted 9 lines
Quoted 6 lines
+ }
+ }
+ if (io_cpu!=queue->io_cpu) {
+queue->io_cpu=io_cpu;
+atomic_inc(&nvme_tcp_cpu_queues[io_cpu]); + }Why is that conditioned ? why not always assign and inc the counter?
Quoted 7 lines
Because we're having the conditional
if (mq_map[cpu] != qid)
above, so technically we might not find the correct mapping. Might be worthwhile doing a
WARN_ON()here, but it might trigger with CPU hotplug...
Writing code that looks like:
if (a != b) {
a = b;
}
is silly.
iiuc It is probably because you initialized the queue->io_cpu to WORK_CPU_UNBOUND and now you don't want to touch it. But it is still kinda ugly.
I suggest that you always select a io_cpu and always account it. You said yourself that if the wq is unbound, it doesn't really matter what the io_cpu is...
And I think that a code comment would help here. its not the most trivial code...
Quoted 27 lines
Quoted 18 lines
Quoted 18 lines
+ dev_dbg(ctrl->ctrl.device, "queue %d: using cpu %d\n",
+ qid,queue->io_cpu);
}
static voidnvme_tcp_tls_done(void *data, int status, key_serial_t pskid)drivers/nvme/host/tcp.c +1
-1@@ -1735,7 +1759,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, queue->sock->sk->sk_allocation = GFP_ATOMIC; queue->sock->sk->sk_use_task_frag = false; - nvme_tcp_set_queue_io_cpu(queue); + queue->io_cpu = WORK_CPU_UNBOUND; queue->request = NULL; queue->data_remaining = 0; queue->ddgst_remaining = 0;@@ -1847,6 +1871,10 @@ static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) kernel_sock_shutdown(queue->sock, SHUT_RDWR); nvme_tcp_restore_sock_ops(queue);Quoted 9 lines
drivers/nvme/host/tcp.c static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) · line 1877
Quoted 5 lines
drivers/nvme/host/tcp.c +4
-0cancel_work_sync(&queue->io_work); + if (queue->io_cpu != WORK_CPU_UNBOUND) { +atomic_dec (&nvme_tcp_cpu_queues[queue->io_cpu]); + queue->io_cpu = WORK_CPU_UNBOUND; + }Does anything change if we still set the
io_cpuinwq_unboundcase? If not, I think we should always do it and simplify the code.
Quoted 2 lines
See above.
WORK_CPU_UNBOUNDmeans that we failed to find the cpu in themq_map. Yes, I know, unlikely, but not impossible.
If you always select a cpu, it should be fine.
Quoted 32 lines
Quoted 23 lines
Quoted 23 lines
drivers/nvme/host/tcp.c +4
-2} static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)@@ -1891,9 +1919,10 @@ static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) nvme_tcp_init_recv_ctx(queue); nvme_tcp_setup_sock_ops(queue); - if (idx) + if (idx) { + nvme_tcp_set_queue_io_cpu(queue); ret = nvmf_connect_io_queue(nctrl, idx); - else + } else ret = nvmf_connect_admin_queue(nctrl); if (!ret) {@@ -2920,6 +2949,7 @@ static struct nvmf_transport_ops nvme_tcp_transport = { static int __init nvme_tcp_init_module(void) { unsigned int wq_flags = WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_SYSFS; + int cpu; BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8); BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72);Quoted 8 lines
drivers/nvme/host/tcp.c static int __init nvme_tcp_init_module(void) · line 2971
Quoted 6 lines
drivers/nvme/host/tcp.c +3
-0@@ -2937,6 +2967,9 @@ static int __init nvme_tcp_init_module(void) if (!nvme_tcp_wq) return -ENOMEM; + for_each_possible_cpu(cpu) + atomic_set(&nvme_tcp_cpu_queues[cpu], 0); +Why?
Quoted 1 line
Don't we need to initialize the atomic counters?
I thought this was the module __exit function, sorry.
PATCH v2 3/3 nvme-tcp: per-controller I/O workqueues
In reply to Hannes ReineckeFrom: Hannes Reinecke <[email protected]>
Implement per-controller I/O workqueues to reduce workqueue contention during I/O and improve I/O performance.
Performance comparison:
baseline rx/tx blk-mq multiple workqueues
4k seq write: 449MiB/s 480MiB/s 524MiB/s 540MiB/s 4k rand write: 410MiB/s 481MiB/s 524MiB/s 539MiB/s
4k seq read: 478MiB/s 481MiB/s 566MiB/s 582MiB/s
4k rand read: 547MiB/s 480MiB/s 511MiB/s 633MiB/s
--- drivers/nvme/host/tcp.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-)
drivers/nvme/host/tcp.c +18 -16
@@ -194,6 +194,7 @@ struct nvme_tcp_ctrl {struct sockaddr_storage src_addr;struct nvme_ctrl ctrl; +struct workqueue_struct *io_wq;struct work_struct err_work;struct delayed_work connect_work;struct nvme_tcp_request async_req;
@@ -202,7 +203,6 @@ struct nvme_tcp_ctrl {static LIST_HEAD (nvme_tcp_ctrl_list);static DEFINE_MUTEX (nvme_tcp_ctrl_mutex); -static struct workqueue_struct *nvme_tcp_wq;static const struct blk_mq_ops nvme_tcp_mq_ops;static const struct blk_mq_ops nvme_tcp_admin_mq_ops;static int nvme_tcp_try_send (struct nvme_tcp_queue *queue);
@@ -410,7 +410,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, }if (last &&nvme_tcp_queue_has_pending (queue)) -queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); +queue_work_on (queue->io_cpu, queue->ctrl->io_wq, &queue->io_work); }static void nvme_tcp_process_req_list (struct nvme_tcp_queue *queue)
@@ -987,7 +987,7 @@ static void nvme_tcp_data_ready(struct sock *sk) queue = sk->sk_user_data;if (likely (queue && queue->rd_enabled) && !test_bit (NVME_TCP_Q_POLLING , &queue->flags)) -queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); +queue_work_on (queue->io_cpu, queue->ctrl->io_wq, &queue->io_work);read_unlock_bh (&sk->sk_callback_lock); }
@@ -999,7 +999,7 @@ static void nvme_tcp_write_space(struct sock *sk) queue = sk->sk_user_data;if (likely (queue &&sk_stream_is_writeable (sk))) {clear_bit (SOCK_NOSPACE , &sk->sk_socket->flags); -queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); +queue_work_on (queue->io_cpu, queue->ctrl->io_wq, &queue->io_work); }read_unlock_bh (&sk->sk_callback_lock); }
@@ -1325,7 +1325,7 @@ static void nvme_tcp_io_work(struct work_struct *w)dev_dbg (queue->ctrl->ctrl.device,"queue %d: queue stall (%u msecs)\n" ,nvme_tcp_queue_id (queue),jiffies_to_msecs (overrun)); -queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); +queue_work_on (queue->io_cpu, queue->ctrl->io_wq, &queue->io_work); }static void nvme_tcp_free_crypto (struct nvme_tcp_queue *queue)
@@ -2486,6 +2486,8 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)nvmf_free_options (nctrl->opts); free_ctrl: +destroy_workqueue (ctrl->io_wq); +kfree (ctrl->queues);kfree (ctrl); }
@@ -2676,7 +2678,7 @@ static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)struct nvme_tcp_queue *queue = hctx->driver_data;if (!llist_empty (&queue->req_list)) -queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); +queue_work_on (queue->io_cpu, queue->ctrl->io_wq, &queue->io_work); } static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
@@ -2812,6 +2814,7 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts) {struct nvme_tcp_ctrl *ctrl; +unsigned int wq_flags =WQ_MEM_RECLAIM |WQ_HIGHPRI |WQ_SYSFS ;int ret; ctrl =kzalloc (sizeof (*ctrl),GFP_KERNEL );
@@ -2883,6 +2886,15 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev,if (ret) goto out_kfree_queues; +if (wq_unbound) + wq_flags |=WQ_UNBOUND ; + ctrl->io_wq =alloc_workqueue ("nvme_tcp_wq_%d" , wq_flags,0 , + ctrl->ctrl.instance); +if (!ctrl->io_wq) { +nvme_put_ctrl (&ctrl->ctrl); +return ERR_PTR (-ENOMEM ); + } +return ctrl; out_kfree_queues:kfree (ctrl->queues);
@@ -2948,7 +2960,6 @@ static struct nvmf_transport_ops nvme_tcp_transport = {static int __init nvme_tcp_init_module (void ) { -unsigned int wq_flags =WQ_MEM_RECLAIM |WQ_HIGHPRI |WQ_SYSFS ;int cpu;BUILD_BUG_ON (sizeof (struct nvme_tcp_hdr ) !=8 );
@@ -2960,13 +2971,6 @@ static int __init nvme_tcp_init_module(void)BUILD_BUG_ON (sizeof (struct nvme_tcp_icresp_pdu ) !=128 );BUILD_BUG_ON (sizeof (struct nvme_tcp_term_pdu ) !=24 ); -if (wq_unbound) - wq_flags |=WQ_UNBOUND ; - - nvme_tcp_wq =alloc_workqueue ("nvme_tcp_wq" , wq_flags,0 ); -if (!nvme_tcp_wq) -return -ENOMEM ; -for_each_possible_cpu (cpu)atomic_set (&nvme_tcp_cpu_queues[cpu],0 );
@@ -2985,8 +2989,6 @@ static void __exit nvme_tcp_cleanup_module(void)nvme_delete_ctrl (&ctrl->ctrl);mutex_unlock (&nvme_tcp_ctrl_mutex);flush_workqueue (nvme_delete_wq); - -destroy_workqueue (nvme_tcp_wq); }module_init (nvme_tcp_init_module);
Signature · 1 line
2.35.3
Sagi Grimberg
maintainerRe: [PATCH 3/3] nvme-tcp: per-controller I/O workqueues
Quoted 11 lines · Hannes Reinecke
From: Hannes Reinecke <[email protected]>
Implement per-controller I/O workqueues to reduce workqueue contention during I/O and improve I/O performance.
Performance comparison:
baseline rx/tx blk-mq multiple workqueues
4k seq write: 449MiB/s 480MiB/s 524MiB/s 540MiB/s 4k rand write: 410MiB/s 481MiB/s 524MiB/s 539MiB/s
4k seq read: 478MiB/s 481MiB/s 566MiB/s 582MiB/s
4k rand read: 547MiB/s 480MiB/s 511MiB/s 633MiB/s
I am still puzzled by this one.
This is for 2 controllers? or more?
It is intresting that the rand read sees higher boost from the seq read. Is this a nature of the SSD? What happens with null_blk ?
CCing Tejun. Is it possible that using two different workqueues for a symmetrical workload is better than a single global workqueue?
Re: [PATCH 3/3] nvme-tcp: per-controller I/O workqueues
Quoted 15 lines · Sagi Grimberg
Quoted 11 lines · Hannes Reinecke
From: Hannes Reinecke <[email protected]>
Implement per-controller I/O workqueues to reduce workqueue contention during I/O and improve I/O performance.
Performance comparison:
baseline rx/tx blk-mq multiple workqueues 4k seq write: 449MiB/s 480MiB/s 524MiB/s 540MiB/s 4k rand write: 410MiB/s 481MiB/s 524MiB/s 539MiB/s 4k seq read: 478MiB/s 481MiB/s 566MiB/s 582MiB/s 4k rand read: 547MiB/s 480MiB/s 511MiB/s 633MiB/s
Quoted 6 lines
I am still puzzled by this one.
This is for 2 controllers? or more?
It is intresting that the rand read sees higher boost from the seq read. Is this a nature of the SSD? What happens withnull_blk?
See the patchset description.
Two controllers, one subsystem.
Quoted 2 lines
CCing Tejun. Is it possible that using two different workqueues for a symmetrical workload is better than a single global workqueue?
Yes, that is the implication.
It might simply due to the fact that the number of workers in a workqueue pool is limited to 512:
workqueue.h: WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */
so by using a workqueue per controller we effectively raise the number
of possible workers. And we're reducing the concurrency between controllers, as each controller can now schedule I/O independent on the other.
Let me check what would happen if I increase MAX_ACTIVE ...
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Sagi Grimberg
maintainerRe: [PATCH 3/3] nvme-tcp: per-controller I/O workqueues
Quoted 16 lines · Hannes Reinecke
Quoted 15 lines · Sagi Grimberg
Quoted 11 lines · Hannes Reinecke
From: Hannes Reinecke <[email protected]>
Implement per-controller I/O workqueues to reduce workqueue contention during I/O and improve I/O performance.
Performance comparison:
baseline rx/tx blk-mq multiple workqueues 4k seq write: 449MiB/s 480MiB/s 524MiB/s 540MiB/s 4k rand write: 410MiB/s 481MiB/s 524MiB/s 539MiB/s 4k seq read: 478MiB/s 481MiB/s 566MiB/s 582MiB/s 4k rand read: 547MiB/s 480MiB/s 511MiB/s 633MiB/s
Quoted 8 lines
Quoted 6 lines
I am still puzzled by this one.
This is for 2 controllers? or more?
It is intresting that the rand read sees higher boost from the seq read. Is this a nature of the SSD? What happens withnull_blk?See the patchset description.
Two controllers, one subsystem.
Can we try with null_blk as well?
Not that I'm dismissing the nvme numbers.
Quoted 7 lines
Quoted 3 lines
CCing Tejun. Is it possible that using two different workqueues for a symmetrical workload is better than a single global workqueue?
Yes, that is the implication.
It might simply due to the fact that the number of workers in a workqueue pool is limited to 512:
Quoted 7 lines
workqueue.h: WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */
so by using a workqueue per controller we effectively raise the number
of possible workers. And we're reducing the concurrency between controllers, as each controller can now schedule I/O independent on the other.Let me check what would happen if I increase
MAX_ACTIVE...
Umm, do you have concurrency of more than 512 active? you have 64 queues in your setup no. Maybe I'm missing something...
Sagi Grimberg
maintainerQuoted 11 lines · Hannes Reinecke
Hi all,
for workloads with a lot of controllers we run into workqueue contention, where the single workqueue is not able to service requests fast enough, leading to spurious I/O errors and connect resets during high load. This patchset improves the situation by improve the fairness between rx and tx scheduling, introducing per-controller workqueues, and distribute the load accoring to the blk-mq cpu mapping. With this we reduce the spurious I/O errors and improve the overall performance for highly contended workloads.
Quoted 1 line
All performance number are derived from the 'tiobench-example.fio'
Did you keep the fio file unmodified? I'd suggest to run it for longer say 60 seconds each workload. 512 MB is a very short benchmark...
Quoted 14 lines · Sagi Grimberg
Quoted 11 lines · Hannes Reinecke
Hi all,
for workloads with a lot of controllers we run into workqueue contention, where the single workqueue is not able to service requests fast enough, leading to spurious I/O errors and connect resets during high load. This patchset improves the situation by improve the fairness between rx and tx scheduling, introducing per-controller workqueues, and distribute the load accoring to the blk-mq cpu mapping. With this we reduce the spurious I/O errors and improve the overall performance for highly contended workloads.
Quoted 4 lines
Quoted 1 line
All performance number are derived from the 'tiobench-example.fio'
Did you keep the fio file unmodified? I'd suggest to run it for longer say 60 seconds each workload. 512 MB is a very short benchmark...
Not for 32 queues :-)
But yeah, I can keep it running for slightly longer.
Not making much progress, mind; your 'softirq' patch definitely speeds up receiving, but seem to messing up the write side such that I'm basically guaranteed to hit I/O timeouts on WRITE :-(
Keep on debugging ...
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Sagi Grimberg
maintainerQuoted 15 lines · Hannes Reinecke
Quoted 14 lines · Sagi Grimberg
Quoted 11 lines · Hannes Reinecke
Hi all,
for workloads with a lot of controllers we run into workqueue contention, where the single workqueue is not able to service requests fast enough, leading to spurious I/O errors and connect resets during high load. This patchset improves the situation by improve the fairness between rx and tx scheduling, introducing per-controller workqueues, and distribute the load accoring to the blk-mq cpu mapping. With this we reduce the spurious I/O errors and improve the overall performance for highly contended workloads.
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
All performance number are derived from the 'tiobench-example.fio'
Did you keep the fio file unmodified? I'd suggest to run it for longer say 60 seconds each workload. 512 MB is a very short benchmark...
Not for 32 queues :-)
But yeah, I can keep it running for slightly longer.
How does the number of queues make a difference?
doesn't it simply write 512MB from 4 threads?
Quoted 4 lines
Not making much progress, mind; your 'softirq' patch definitely speeds up receiving, but seem to messing up the write side such that I'm basically guaranteed to hit I/O timeouts on WRITE :-(
Keep on debugging ...
Hannes, I think that now that we established that rx can starve tx, we must pace rx, so if we do something like softirq rx, it must be an initial batch, limited, and once we exhaust it, we schedule a workqueue to continue processing. I'd also leave it alone for now, we'll add it once we have a good understanding of what is going on...
Sagi Grimberg
maintainerQuoted 16 lines · Hannes Reinecke
Hi all,
for workloads with a lot of controllers we run into workqueue contention, where the single workqueue is not able to service requests fast enough, leading to spurious I/O errors and connect resets during high load. This patchset improves the situation by improve the fairness between rx and tx scheduling, introducing per-controller workqueues, and distribute the load accoring to the blk-mq cpu mapping. With this we reduce the spurious I/O errors and improve the overall performance for highly contended workloads.
All performance number are derived from the 'tiobench-example.fio' sample from the fio sources, running on a 96 core machine with one subsystem and two paths, each path exposing 32 queues. Backend is nvmet using an Intel DC P3700 NVMe SSD.
Quoted 4 lines
Changes to the initial submission:
- Make the changes independent from the 'wq_unbound' parameter
- Drop changes to the workqueue
- Add patch to improve rx/tx fairness
Hey Hannes, were you able to make progress here?
Thought I'd ask...
Quoted 19 lines · Sagi Grimberg
Quoted 16 lines · Hannes Reinecke
Hi all,
for workloads with a lot of controllers we run into workqueue contention, where the single workqueue is not able to service requests fast enough, leading to spurious I/O errors and connect resets during high load. This patchset improves the situation by improve the fairness between rx and tx scheduling, introducing per-controller workqueues, and distribute the load accoring to the blk-mq cpu mapping. With this we reduce the spurious I/O errors and improve the overall performance for highly contended workloads.
All performance number are derived from the 'tiobench-example.fio' sample from the fio sources, running on a 96 core machine with one subsystem and two paths, each path exposing 32 queues. Backend is nvmet using an Intel DC P3700 NVMe SSD.
Quoted 7 lines
Quoted 4 lines
Changes to the initial submission:
- Make the changes independent from the 'wq_unbound' parameter
- Drop changes to the workqueue
- Add patch to improve rx/tx fairnessHey Hannes, were you able to make progress here?
Thought I'd ask...
Oh, well. Things are going back and forth, and occasionally sideways. But I'm having something resembling a patchset; most of it is actually debugging stuff allowing to analyse latencies. Currently trying to get some performance numbers, but should be ready shortly.
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich