nvme/pci: Use a flag for polled queues
PATCH 1/8 nvme/pci: Use a flag for polled queues
A negative value for the cq_vector used to mean the queue is either disabled or a polled queue. However, we have a queue enabled flag, so the cq_vector had been serving double duty.
Don't overload the meaning of cq_vector. Use a flag specific to the polled queues instead.
--- drivers/nvme/host/pci.c | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-)
drivers/nvme/host/pci.c +12 -19
@@ -189,7 +189,7 @@ struct nvme_queue {dma_addr_t cq_dma_addr;u32 __iomem *q_db;u16 q_depth; -s16 cq_vector; +u16 cq_vector;u16 sq_tail;u16 last_sq_tail;u16 cq_head;
@@ -200,6 +200,7 @@ struct nvme_queue {#define NVMEQ_ENABLED 0#define NVMEQ_SQ_CMB 1#define NVMEQ_DELETE_ERROR 2 +#define NVMEQ_POLLED 3u32 *dbbuf_sq_db;u32 *dbbuf_cq_db;u32 *dbbuf_sq_ei;
@@ -1141,7 +1142,7 @@ static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,struct nvme_command c;int flags =NVME_QUEUE_PHYS_CONTIG ; -if (vector !=-1 ) +if (!test_bit (NVMEQ_POLLED , &nvmeq->flags)) flags |=NVME_CQ_IRQ_ENABLED ;/*
@@ -1154,10 +1155,7 @@ static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, c.create_cq.cqid =cpu_to_le16 (qid); c.create_cq.qsize =cpu_to_le16 (nvmeq->q_depth -1 ); c.create_cq.cq_flags =cpu_to_le16 (flags); -if (vector !=-1 ) - c.create_cq.irq_vector =cpu_to_le16 (vector); -else - c.create_cq.irq_vector =0 ; + c.create_cq.irq_vector =cpu_to_le16 (vector);return nvme_submit_sync_cmd (dev->ctrl.admin_q, &c,NULL ,0 ); }
@@ -1399,10 +1397,8 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq) nvmeq->dev->online_queues--;if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)blk_mq_quiesce_queue (nvmeq->dev->ctrl.admin_q); -if (nvmeq->cq_vector ==-1 ) -return 0 ; -pci_free_irq (to_pci_dev (nvmeq->dev->dev), nvmeq->cq_vector, nvmeq); - nvmeq->cq_vector =-1 ; +if (!test_and_clear_bit (NVMEQ_POLLED , &nvmeq->flags)) +pci_free_irq (to_pci_dev (nvmeq->dev->dev), nvmeq->cq_vector, nvmeq);return 0 ; }
@@ -1496,7 +1492,6 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) nvmeq->q_db = &dev->dbs[qid *2 * dev->db_stride]; nvmeq->q_depth = depth; nvmeq->qid = qid; - nvmeq->cq_vector =-1 ; dev->ctrl.queue_count++;return 0 ;
@@ -1541,7 +1536,7 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) {struct nvme_dev *dev = nvmeq->dev;int result; -s16 vector; +u16 vector =0 ;clear_bit (NVMEQ_DELETE_ERROR , &nvmeq->flags);
@@ -1552,7 +1547,7 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)if (!polled) vector = dev->num_vecs ==1 ?0 : qid;else - vector =-1 ; +set_bit (NVMEQ_POLLED , &nvmeq->flags); result =adapter_alloc_cq (dev, qid, nvmeq, vector);if (result)
@@ -1567,7 +1562,8 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) nvmeq->cq_vector = vector;nvme_init_queue (nvmeq, qid); -if (vector !=-1 ) { +if (!polled) { + nvmeq->cq_vector = vector; result =queue_request_irq (nvmeq);if (result <0 ) goto release_sq;
@@ -1577,7 +1573,6 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)return result; release_sq: - nvmeq->cq_vector =-1 ; dev->online_queues--;adapter_delete_sq (dev, qid); release_cq:
@@ -1733,7 +1728,7 @@ static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)nvme_init_queue (nvmeq,0 ); result =queue_request_irq (nvmeq);if (result) { - nvmeq->cq_vector =-1 ; + dev->online_queues--;return result; }
@@ -2214,10 +2209,8 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)* number of interrupts. */ result =queue_request_irq (adminq); -if (result) { - adminq->cq_vector =-1 ; +if (result)return result; - }set_bit (NVMEQ_ENABLED , &adminq->flags); result =nvme_create_io_queues (dev);
Signature · 1 line
2.14.4
PATCH 2/8 nvme/pci: Don't poll polled queues in timeout
Polled queues are already polled. If a timeout occurs on such a queue, escalate the error recovery immediately.
--- drivers/nvme/host/pci.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-)
drivers/nvme/host/pci.c +5 -15
@@ -1077,20 +1077,9 @@ static int nvme_poll_irqdisable(struct nvme_queue *nvmeq, unsigned int tag)u16 start, end;int found; -/* -* For a poll queue we need to protect against the polling thread -* using the CQ lock. For normal interrupt driven threads we have -* to disable the interrupt to avoid racing with it. -*/ -if (nvmeq->cq_vector ==-1 ) { -spin_lock (&nvmeq->cq_poll_lock); - found =nvme_process_cq (nvmeq, &start, &end, tag); -spin_unlock (&nvmeq->cq_poll_lock); - }else { -disable_irq (pci_irq_vector (pdev, nvmeq->cq_vector)); - found =nvme_process_cq (nvmeq, &start, &end, tag); -enable_irq (pci_irq_vector (pdev, nvmeq->cq_vector)); - } +disable_irq (pci_irq_vector (pdev, nvmeq->cq_vector)); + found =nvme_process_cq (nvmeq, &start, &end, tag); +enable_irq (pci_irq_vector (pdev, nvmeq->cq_vector));nvme_complete_cqes (nvmeq, start, end);return found;
@@ -1284,7 +1273,8 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)/* * Did we miss an interrupt? */ -if (nvme_poll_irqdisable (nvmeq, req->tag)) { +if (!test_bit (NVMEQ_POLLED , &nvmeq->flags) && +nvme_poll_irqdisable (nvmeq, req->tag)) {dev_warn (dev->ctrl.device,"I/O %d QID %d timeout, completion polled\n" , req->tag, nvmeq->qid);
Signature · 1 line
2.14.4
Christoph Hellwig
maintainerPATCH 2/8 nvme/pci: Don't poll polled queues in timeout
Quoted 2 lines
Polled queues are already polled. If a timeout occurs on such a queue, escalate the error recovery immediately.
But don't we still need to poll them when tearing down the queues?
Sagi Grimberg
maintainerPATCH 2/8 nvme/pci: Don't poll polled queues in timeout
In reply to Keith BuschQuoted 24 lines
Polled queues are already polled. If a timeout occurs on such a queue, escalate the error recovery immediately.
Signed-off-by Keith Busch <keith.busch at intel.com>--- drivers/nvme/host/pci.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-)drivers/nvme/host/pci.c +0
-7@@ -1077,20 +1077,9 @@ static int nvme_poll_irqdisable(struct nvme_queue *nvmeq, unsigned int tag)u16 start, end;int found; -/* -* For a poll queue we need to protect against the polling thread -* using the CQ lock. For normal interrupt driven threads we have -* to disable the interrupt to avoid racing with it. -*/ -if (nvmeq->cq_vector ==-1 ) { -spin_lock (&nvmeq->cq_poll_lock);
drivers/nvme/host/pci.c static int nvme_poll_irqdisable(struct nvme_queue *nvmeq, unsigned int tag) · line 1082
Quoted 10 lines
drivers/nvme/host/pci.c +3
-7- found =nvme_process_cq (nvmeq, &start, &end, tag); -spin_unlock (&nvmeq->cq_poll_lock); - }else { -disable_irq (pci_irq_vector (pdev, nvmeq->cq_vector)); - found =nvme_process_cq (nvmeq, &start, &end, tag); -enable_irq (pci_irq_vector (pdev, nvmeq->cq_vector)); - } +disable_irq (pci_irq_vector (pdev, nvmeq->cq_vector)); + found =nvme_process_cq (nvmeq, &start, &end, tag); +enable_irq (pci_irq_vector (pdev, nvmeq->cq_vector));
Why do polled queues need irq disable?
Quoted 14 lines
drivers/nvme/host/pci.c +2
-1nvme_complete_cqes (nvmeq, start, end);return found;@@ -1284,7 +1273,8 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)/* * Did we miss an interrupt? */ -if (nvme_poll_irqdisable (nvmeq, req->tag)) { +if (!test_bit (NVMEQ_POLLED , &nvmeq->flags) && +nvme_poll_irqdisable (nvmeq, req->tag)) {dev_warn (dev->ctrl.device,"I/O %d QID %d timeout, completion polled\n" , req->tag, nvmeq->qid);
PATCH 3/8 nvme/pci: Remove tag check in nvme_process_cq
In reply to Keith BuschSimplify processing completions by removing the check for the optional search tag. The timeout handling was the only place using this, but we can check the request state directly instead of searching the completion entries. There is a chance the command was not actually polled in this implementation, but we don't want to escalate error recovery in that case anyway, and the warning is still informative either way.
--- drivers/nvme/host/pci.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-)
drivers/nvme/host/pci.c +16 -17
@@ -1017,14 +1017,13 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq) }static inline int nvme_process_cq (struct nvme_queue *nvmeq,u16 *start, -u16 *end,unsigned int tag) +u16 *end) {int found =0 ; *start = nvmeq->cq_head;while (nvme_cqe_pending (nvmeq)) { -if (tag ==-1U || nvmeq->cqes[nvmeq->cq_head].command_id == tag) - found++; + found++;nvme_update_cq_head (nvmeq); } *end = nvmeq->cq_head;
@@ -1047,7 +1046,7 @@ static irqreturn_t nvme_irq(int irq, void *data)rmb ();if (nvmeq->cq_head != nvmeq->last_cq_head) ret =IRQ_HANDLED ; -nvme_process_cq (nvmeq, &start, &end,-1 ); +nvme_process_cq (nvmeq, &start, &end); nvmeq->last_cq_head = nvmeq->cq_head;wmb ();
@@ -1071,18 +1070,16 @@ static irqreturn_t nvme_irq_check(int irq, void *data)* Poll for completions any queue, including those not dedicated to polling. * Can be called from any context. */ -static int nvme_poll_irqdisable (struct nvme_queue *nvmeq,unsigned int tag) +static void nvme_poll_irqdisable (struct nvme_queue *nvmeq) {struct pci_dev *pdev =to_pci_dev (nvmeq->dev->dev);u16 start, end; -int found;disable_irq (pci_irq_vector (pdev, nvmeq->cq_vector)); - found =nvme_process_cq (nvmeq, &start, &end, tag); +nvme_process_cq (nvmeq, &start, &end);enable_irq (pci_irq_vector (pdev, nvmeq->cq_vector));nvme_complete_cqes (nvmeq, start, end); -return found; }static int nvme_poll (struct blk_mq_hw_ctx *hctx)
@@ -1095,7 +1092,7 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx)return 0 ;spin_lock (&nvmeq->cq_poll_lock); - found =nvme_process_cq (nvmeq, &start, &end,-1 ); + found =nvme_process_cq (nvmeq, &start, &end);spin_unlock (&nvmeq->cq_poll_lock);nvme_complete_cqes (nvmeq, start, end);
@@ -1273,12 +1270,14 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)/* * Did we miss an interrupt? */ -if (!test_bit (NVMEQ_POLLED , &nvmeq->flags) && -nvme_poll_irqdisable (nvmeq, req->tag)) { -dev_warn (dev->ctrl.device, -"I/O %d QID %d timeout, completion polled\n" , - req->tag, nvmeq->qid); -return BLK_EH_DONE ; +if (!test_bit (NVMEQ_POLLED , &nvmeq->flags)) { +nvme_poll_irqdisable (nvmeq); +if (blk_mq_rq_state (req) !=MQ_RQ_IN_FLIGHT ) { +dev_warn (dev->ctrl.device, +"I/O %d QID %d timeout, completion polled\n" , + req->tag, nvmeq->qid); +return BLK_EH_DONE ; + } }/*
@@ -1409,7 +1408,7 @@ static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)else nvme_disable_ctrl (&dev->ctrl, dev->ctrl.cap); -nvme_poll_irqdisable (nvmeq,-1 ); +nvme_poll_irqdisable (nvmeq); } static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
@@ -2286,7 +2285,7 @@ static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode)/* handle any remaining CQEs */ if (opcode == nvme_admin_delete_cq && !test_bit (NVMEQ_DELETE_ERROR , &nvmeq->flags)) -nvme_poll_irqdisable (nvmeq,-1 ); +nvme_poll_irqdisable (nvmeq); sent--;if (nr_queues)
Signature · 1 line
2.14.4
Christoph Hellwig
maintainerPATCH 3/8 nvme/pci: Remove tag check in nvme_process_cq
Quoted 8 lines · Keith Busch
Simplify processing completions by removing the check for the optional search tag. The timeout handling was the only place using this, but we can check the request state directly instead of searching the completion entries. There is a chance the command was not actually polled in this implementation, but we don't want to escalate error recovery in that case anyway, and the warning is still informative either way.
Signed-off-by Keith Busch <keith.busch at intel.com>
Looks good:
Sagi Grimberg
maintainerPATCH 3/8 nvme/pci: Remove tag check in nvme_process_cq
In reply to Keith BuschThis is good,
PATCH 4/8 nvme/pci: Remove last_cq_seen
In reply to Keith BuschThe driver had been saving the last CQ seen in case polling competes with an interrupt handler in order to defeat spurious interrupt detection. Polling only applies to special queues that don't have interrupt handlers now, so we don't need to save this.
--- drivers/nvme/host/pci.c | 4 ---- 1 file changed, 4 deletions(-)
drivers/nvme/host/pci.c +0 -4
@@ -193,7 +193,6 @@ struct nvme_queue {u16 sq_tail;u16 last_sq_tail;u16 cq_head; -u16 last_cq_head;u16 qid;u8 cq_phase;unsigned long flags;
@@ -1044,10 +1043,7 @@ static irqreturn_t nvme_irq(int irq, void *data)* the irq handler, even if that was on another CPU. */ rmb (); -if (nvmeq->cq_head != nvmeq->last_cq_head) - ret =IRQ_HANDLED ;nvme_process_cq (nvmeq, &start, &end); - nvmeq->last_cq_head = nvmeq->cq_head;wmb ();if (start != end) {
Signature · 1 line
2.14.4
Christoph Hellwig
maintainerPATCH 4/8 nvme/pci: Remove last_cq_seen
Quoted 13 lines
drivers/nvme/host/pci.c +0
-1@@ -193,7 +193,6 @@ struct nvme_queue {u16 sq_tail;u16 last_sq_tail;u16 cq_head; -u16 last_cq_head;u16 qid;u8 cq_phase;unsigned long flags;
drivers/nvme/host/pci.c static irqreturn_t nvme_irq(int irq, void *data) · line 1049
Quoted 10 lines
drivers/nvme/host/pci.c +0
-3@@ -1044,10 +1043,7 @@ static irqreturn_t nvme_irq(int irq, void *data)* the irq handler, even if that was on another CPU. */ rmb (); -if (nvmeq->cq_head != nvmeq->last_cq_head) - ret =IRQ_HANDLED ;nvme_process_cq (nvmeq, &start, &end); - nvmeq->last_cq_head = nvmeq->cq_head;wmb ();if (start != end) {
This looks good, but we can also kill the ret variable now, and simplify the tail a bit to:
if (start == end) return IRQ_NONE; nvme_complete_cqes(nvmeq, start, end); return IRQ_HANDLED;
Sagi Grimberg
maintainerPATCH 4/8 nvme/pci: Remove last_cq_seen
Quoted 27 lines
Quoted 13 lines
index
45db4a39795e..62f29ab93f5a100644drivers/nvme/host/pci.c +0
-1@@ -193,7 +193,6 @@ struct nvme_queue {u16 sq_tail;u16 last_sq_tail;u16 cq_head; -u16 last_cq_head;u16 qid;u8 cq_phase;unsigned long flags;drivers/nvme/host/pci.c static irqreturn_t nvme_irq(int irq, void *data) · line 1049
Quoted 10 lines
drivers/nvme/host/pci.c +0
-3@@ -1044,10 +1043,7 @@ static irqreturn_t nvme_irq(int irq, void *data)* the irq handler, even if that was on another CPU. */ rmb (); -if (nvmeq->cq_head != nvmeq->last_cq_head) - ret =IRQ_HANDLED ;nvme_process_cq (nvmeq, &start, &end); - nvmeq->last_cq_head = nvmeq->cq_head;wmb ();if (start != end) {This looks good, but we can also kill the ret variable now, and simplify the tail a bit to:
Quoted 4 lines
if (start == end) return IRQ_NONE; nvme_complete_cqes(nvmeq, start, end); return IRQ_HANDLED;
Why not just look at what nvme_process_cq is returning?
Christoph Hellwig
maintainerPATCH 4/8 nvme/pci: Remove last_cq_seen
Quoted 9 lines · Sagi Grimberg
Quoted 7 lines
This looks good, but we can also kill the ret variable now, and simplify the tail a bit to:
if (start == end) return IRQ_NONE; nvme_complete_cqes(nvmeq, start, end); return IRQ_HANDLED;Why not just look at what
nvme_process_cqis returning?
Yes, that might be even simpler.
PATCH 5/8 nvme/pci: Remove last_sq_tail
In reply to Keith BuschWe don't allocate enough tags to wrap the submission queue ring. Remove the checks for this condition and shrink struct nvme_queue.
--- drivers/nvme/host/pci.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-)
drivers/nvme/host/pci.c +4 -16
@@ -191,7 +191,6 @@ struct nvme_queue {u16 q_depth;u16 cq_vector;u16 sq_tail; -u16 last_sq_tail;u16 cq_head;u16 qid;u8 cq_phase;
@@ -514,21 +513,11 @@ static int nvme_pci_map_queues(struct blk_mq_tag_set *set)/* * Write sq tail if we are asked to, or if the next command would wrap. */ -static inline void nvme_write_sq_db (struct nvme_queue *nvmeq,bool write_sq) +static inline void nvme_write_sq_db (struct nvme_queue *nvmeq) { -if (!write_sq) { -u16 next_tail = nvmeq->sq_tail +1 ; - -if (next_tail == nvmeq->q_depth) - next_tail =0 ; -if (next_tail != nvmeq->last_sq_tail) -return ; - } -if (nvme_dbbuf_update_and_check_event (nvmeq->sq_tail, nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))writel (nvmeq->sq_tail, nvmeq->q_db); - nvmeq->last_sq_tail = nvmeq->sq_tail; }/**
@@ -544,7 +533,8 @@ static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,memcpy (&nvmeq->sq_cmds[nvmeq->sq_tail], cmd,sizeof (*cmd));if (++nvmeq->sq_tail == nvmeq->q_depth) nvmeq->sq_tail =0 ; -nvme_write_sq_db (nvmeq, write_sq); +if (write_sq) +nvme_write_sq_db (nvmeq);spin_unlock (&nvmeq->sq_lock); }
@@ -553,8 +543,7 @@ static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx)struct nvme_queue *nvmeq = hctx->driver_data;spin_lock (&nvmeq->sq_lock); -if (nvmeq->sq_tail != nvmeq->last_sq_tail) -nvme_write_sq_db (nvmeq, true); +nvme_write_sq_db (nvmeq);spin_unlock (&nvmeq->sq_lock); }
@@ -1507,7 +1496,6 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)struct nvme_dev *dev = nvmeq->dev; nvmeq->sq_tail =0 ; - nvmeq->last_sq_tail =0 ; nvmeq->cq_head =0 ; nvmeq->cq_phase =1 ; nvmeq->q_db = &dev->dbs[qid *2 * dev->db_stride];
Signature · 1 line
2.14.4
Christoph Hellwig
maintainerPATCH 5/8 nvme/pci: Remove last_sq_tail
Quoted 2 lines · Keith Busch
We don't allocate enough tags to wrap the submission queue ring. Remove the checks for this condition and shrink struct
nvme_queue.
Hmm, I don't see how this is related to wrapping, more about avoiding SQ doorbell writes until we actually commit the current batch of I/O.
It originated from:
ommit 04f3eafda6e05adc56afed4d3ae6e24aaa429058
Author: Jens Axboe <axboe at kernel.dk> Date: Thu Nov 29 10:02:29 2018 -0700
nvme: implement mq_ops->commit_rqs() hook
Keith Busch
maintainerPATCH 5/8 nvme/pci: Remove last_sq_tail
Quoted 7 lines · Christoph Hellwig
Quoted 2 lines · Keith Busch
We don't allocate enough tags to wrap the submission queue ring. Remove the checks for this condition and shrink struct
nvme_queue.Hmm, I don't see how this is related to wrapping, more about avoiding SQ doorbell writes until we actually commit the current batch of I/O.
Quoted 7 lines
It originated from:
ommit
04f3eafda6e05adc56afed4d3ae6e24aaa429058Author: Jens Axboe <axboe at kernel.dk> Date: Thu Nov 29 10:02:29 2018 -0700nvme: implement mq_ops->commit_rqs() hook
Right, the batched doorblell writes is the commit, and this comment explains the wrap detection requirement that it brought with it:
"Write sq tail if we are asked to, or if the next command would wrap."
Batched DB writes would need this if we could wrap, and the wrap detection has to compare the previously wrtten sq tail the next entry. We just don't have enough tags to do that, so it's not necessary to check.
I should have changed that comment in this patch, though.
Sagi Grimberg
maintainerPATCH 5/8 nvme/pci: Remove last_sq_tail
Quoted 18 lines
Quoted 7 lines
Quoted 3 lines
We don't allocate enough tags to wrap the submission queue ring. Remove the checks for this condition and shrink struct
nvme_queue.Hmm, I don't see how this is related to wrapping, more about avoiding SQ doorbell writes until we actually commit the current batch of I/O.
Quoted 7 lines
It originated from:
ommit
04f3eafda6e05adc56afed4d3ae6e24aaa429058Author: Jens Axboe <axboe at kernel.dk> Date: Thu Nov 29 10:02:29 2018 -0700nvme: implement mq_ops->commit_rqs() hookRight, the batched doorblell writes is the commit, and this comment explains the wrap detection requirement that it brought with it:
Quoted 7 lines
"Write sq tail if we are asked to, or if the next command would wrap."
Batched DB writes would need this if we could wrap, and the wrap detection has to compare the previously wrtten sq tail the next entry. We just don't have enough tags to do that, so it's not necessary to check.
I should have changed that comment in this patch, though.
The code writes DB when the tail == q_depth, which is a wrap. Is this DB write redundant? If not than it should be stated as this is changing the logic.
PATCH 6/8 nvme/pci: Remove q_dmadev from nvme_queue
In reply to Keith BuschWe don't need to save the dma device as it's not used in the hot path and hasn't in a long time. Shrink the struct nvme_queue removing this unnecessary member.
--- drivers/nvme/host/pci.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
drivers/nvme/host/pci.c +3 -5
@@ -177,7 +177,6 @@ static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)* commands and one for I/O commands). */ struct nvme_queue { -struct device *q_dmadev;struct nvme_dev *dev;spinlock_t sq_lock;struct nvme_command *sq_cmds;
@@ -1336,16 +1335,16 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)static void nvme_free_queue (struct nvme_queue *nvmeq) { -dma_free_coherent (nvmeq->q_dmadev,CQ_SIZE (nvmeq->q_depth), +dma_free_coherent (nvmeq->dev->dev,CQ_SIZE (nvmeq->q_depth), (void *)nvmeq->cqes, nvmeq->cq_dma_addr);if (!nvmeq->sq_cmds)return ;if (test_and_clear_bit (NVMEQ_SQ_CMB , &nvmeq->flags)) { -pci_free_p2pmem (to_pci_dev (nvmeq->q_dmadev), +pci_free_p2pmem (to_pci_dev (nvmeq->dev->dev), nvmeq->sq_cmds,SQ_SIZE (nvmeq->q_depth)); }else { -dma_free_coherent (nvmeq->q_dmadev,SQ_SIZE (nvmeq->q_depth), +dma_free_coherent (nvmeq->dev->dev,SQ_SIZE (nvmeq->q_depth), nvmeq->sq_cmds, nvmeq->sq_dma_addr); } }
@@ -1457,7 +1456,6 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)if (nvme_alloc_sq_cmds (dev, nvmeq, qid, depth)) goto free_cqdma; - nvmeq->q_dmadev = dev->dev; nvmeq->dev = dev;spin_lock_init (&nvmeq->sq_lock);spin_lock_init (&nvmeq->cq_poll_lock);
Signature · 1 line
2.14.4
Christoph Hellwig
maintainerPATCH 6/8 nvme/pci: Remove q_dmadev from nvme_queue
Quoted 3 lines · Keith Busch
We don't need to save the dma device as it's not used in the hot path and hasn't in a long time. Shrink the struct
nvme_queueremoving this unnecessary member.
Looks good,
Sagi Grimberg
maintainerPATCH 6/8 nvme/pci: Remove q_dmadev from nvme_queue
In reply to Keith BuschPATCH 6/8 nvme/pci: Remove q_dmadev from nvme_queue
In reply to Keith BuschThanks,
applied to nvme-5.2.
PATCH 7/8 nvme/pci: Remove volatile from cqe
In reply to Keith BuschThe cqe isn't volatile once we confirm the phase bit. Remove the volatile keyword and let the compiler optimize out an unnecessary additional read to the command id.
--- drivers/nvme/host/pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/nvme/host/pci.c +1 -1
@@ -956,7 +956,7 @@ static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)static inline void nvme_handle_cqe (struct nvme_queue *nvmeq,u16 idx) { -volatile struct nvme_completion *cqe = &nvmeq->cqes[idx]; +struct nvme_completion *cqe = (struct nvme_completion *)&nvmeq->cqes[idx];struct request *req;if (unlikely (cqe->command_id >= nvmeq->q_depth)) {
Signature · 1 line
2.14.4
Christoph Hellwig
maintainerPATCH 7/8 nvme/pci: Remove volatile from cqe
Quoted 3 lines · Keith Busch
The cqe isn't volatile once we confirm the phase bit. Remove the volatile keyword and let the compiler optimize out an unnecessary additional read to the command id.
Hmm. I've never been happy about how we used volatile to start with, but I'm not sure more casts sort out this problem.
Wouldn't the right fix be to drop the volatile entirely and force a strong barrier before first reading the phase bit?
Sagi Grimberg
maintainerPATCH 7/8 nvme/pci: Remove volatile from cqe
Quoted 10 lines
Quoted 4 lines
The cqe isn't volatile once we confirm the phase bit. Remove the volatile keyword and let the compiler optimize out an unnecessary additional read to the command id.
Hmm. I've never been happy about how we used volatile to start with, but I'm not sure more casts sort out this problem.
Wouldn't the right fix be to drop the volatile entirely and force a strong barrier before first reading the phase bit?
Is there a fundamental difference efficiency-wise?
Christoph Hellwig
maintainerPATCH 7/8 nvme/pci: Remove volatile from cqe
Quoted 5 lines · Sagi Grimberg
Quoted 5 lines
Quoted 4 lines
The cqe isn't volatile once we confirm the phase bit. Remove the volatile keyword and let the compiler optimize out an unnecessary additional read to the command id.
Quoted 7 lines
Quoted 5 lines
Hmm. I've never been happy about how we used volatile to start with, but I'm not sure more casts sort out this problem.
Wouldn't the right fix be to drop the volatile entirely and force a strong barrier before first reading the phase bit?
Is there a fundamental difference efficiency-wise?
I'm not sure. The latter is just more our usual Linux style.
PATCH 8/8 nvme/pci: Remove unused nvme_iod member
In reply to Keith Busch--- drivers/nvme/host/pci.c | 2 -- 1 file changed, 2 deletions(-)
drivers/nvme/host/pci.c +0 -2
@@ -218,7 +218,6 @@ struct nvme_iod {int aborted;int npages;/* In the PRP list. 0 means small pool in use */ int nents;/* Used in scatterlist */ -int length;/* Of data, in bytes */ dma_addr_t first_dma;struct scatterlist meta_sg;/* metadata requires single contiguous buffer */ struct scatterlist *sg;
@@ -591,7 +590,6 @@ static blk_status_t nvme_init_iod(struct request *rq, struct nvme_dev *dev) iod->aborted =0 ; iod->npages =-1 ; iod->nents =0 ; - iod->length = size;return BLK_STS_OK ; }
Signature · 1 line
2.14.4
Christoph Hellwig
maintainerPATCH 8/8 nvme/pci: Remove unused nvme_iod member
Looks good. So good that I have my own version of it:
http://git.infradead.org/users/hch/block.git/commitdiff/26cabe40a2225d3f9b78da0082f150036b150e03
Sagi Grimberg
maintainerPATCH 8/8 nvme/pci: Remove unused nvme_iod member
In reply to Keith BuschPATCH 8/8 nvme/pci: Remove unused nvme_iod member
In reply to Keith BuschThanks,
applied to nvme-5.2.
Christoph Hellwig
maintainerPATCH 1/8 nvme/pci: Use a flag for polled queues
In reply to Keith BuschQuoted 6 lines · Keith Busch
A negative value for the
cq_vectorused to mean the queue is either disabled or a polled queue. However, we have a queue enabled flag, so thecq_vectorhad been serving double duty.Don't overload the meaning of
cq_vector. Use a flag specific to the polled queues instead.
Looks good,
a few nitpicks below:
drivers/nvme/host/pci.c static int nvme_suspend_queue(struct nvme_queue *nvmeq) · line 1402
Quoted 4 lines
drivers/nvme/host/pci.c +2
-1- nvmeq->cq_vector =-1 ; +if (!test_and_clear_bit (NVMEQ_POLLED , &nvmeq->flags)) +pci_free_irq (to_pci_dev (nvmeq->dev->dev), nvmeq->cq_vector, nvmeq);return 0 ;
This creates an > 80 char line.
But given that the nvme_suspend_queue return value is always ignored you could throw in a prep patch to remove the return value and just do an early return for the poll case here.
drivers/nvme/host/pci.c static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) · line 1550
Quoted 6 lines
drivers/nvme/host/pci.c +1
-1@@ -1552,7 +1547,7 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)if (!polled) vector = dev->num_vecs ==1 ?0 : qid;else - vector =-1 ; +set_bit (NVMEQ_POLLED , &nvmeq->flags);
Between the inversion and the ? : this looks pretty obsfucated. Can we untangle this to:
if (polled) set_bit(NVMEQ_POLLED, &nvmeq->flags); else if (dev->num_vecs == 1) vector = 0; else vector = qid;
Or we might just kill the polled argument and the local vector variable entirely, set the NVMEQ_POLLED flag in nvme_create_io_queues and calculate the vector inside adapter_alloc_cq, which would also lose the vector argument.
Sagi Grimberg
maintainerPATCH 1/8 nvme/pci: Use a flag for polled queues
In reply to Keith BuschThis ended up in my spam folder
(hope its not a hint to the review ;))
Looks good,
PATCH 1/8 nvme/pci: Use a flag for polled queues
In reply to Keith BuschThanks,
applied to nvme-5.2.