Skip to contentSkip to conversation

blk-mq: fix blk_mq_alloc_request_hctx

PATCH Superseded Ming Lei 4 participants 11 messages Jul 21, 2021 – Jul 22, 2021

Subsystems (4)

v5 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

Ming Lei

author
#0 lore raw

v5 cover blk-mq: fix blk_mq_alloc_request_hctx

Hi,

blk_mq_alloc_request_hctx() is used by NVMe fc/rdma/tcp/loop to connect io queue. Also the sw ctx is chosen as the 1st online cpu in hctx->cpumask. However, all cpus in hctx->cpumask may be offline.

This usage model isn't well supported by blk-mq which supposes allocator is always done on one online CPU in hctx->cpumask. This assumption is related with managed irq, which also requires blk-mq to drain inflight request in this hctx when the last cpu in hctx->cpumask is going to offline.

However, NVMe fc/rdma/tcp/loop don't use managed irq, so we should allow them to ask for request allocation when the specified hctx is inactive (all cpus in hctx->cpumask are offline). Fix blk_mq_alloc_request_hctx() by allowing to allocate request when all CPUs of this hctx are offline.

Wen Xiong has verified V4 in her nvmef test.

V5:
- take John Garry's suggestion to replace device field with new helper of device_has_managed_msi_irq()
V4:
- remove patches for cleanup queue map helpers
- take Christoph's suggestion to add field into 'struct device' for describing if managed irq is allocated from one device

V3:
- cleanup map queues helpers, and remove pci/virtio/rdma queue helpers
- store use managed irq info into qmap

V2:
- use flag of BLK_MQ_F_MANAGED_IRQ
- pass BLK_MQ_F_MANAGED_IRQ from driver explicitly
- kill BLK_MQ_F_STACKING

Ming Lei (3):
driver core: add device_has_managed_msi_irq
blk-mq: mark if one queue map uses managed irq
blk-mq: don't deactivate hctx if managed irq isn't used

 block/blk-mq-pci.c                     |  1 +
 block/blk-mq-rdma.c                    |  3 +++
 block/blk-mq-virtio.c                  |  1 +
 block/blk-mq.c                         | 27 ++++++++++++++++----------
 block/blk-mq.h                         |  8 ++++++++
 drivers/base/core.c                    | 15 ++++++++++++++
 drivers/scsi/hisi_sas/hisi_sas_v2_hw.c |  1 +
 include/linux/blk-mq.h                 |  3 ++-
 include/linux/device.h                 |  2 ++
 9 files changed, 50 insertions(+), 11 deletions(-)
Signature · 1 line
2.31.1
author
#1 lore raw

PATCH v5 1/3 driver core: add device_has_managed_msi_irq

irq vector allocation with managed affinity may be used by driver, and blk-mq needs this info for draining queue because genirq core will shutdown managed irq when all CPUs in the affinity mask are offline.

The info of using managed irq is often produced by drivers, and it is consumed by blk-mq, so different subsystems are involved in this info flow.

Address this issue by adding one helper of device_has_managed_msi_irq() which is suggested by John Garry.

---
 drivers/base/core.c    | 15 +++++++++++++++
 include/linux/device.h |  2 ++
 2 files changed, 17 insertions(+)
drivers/base/core.c +15 -0
index cadcade65825..41daf9fabdfb 100644
@@ -29,6 +29,7 @@
 #include <linux/sched/mm.h>
 #include <linux/sysfs.h>
 #include <linux/dma-map-ops.h> /* for dma_default_coherent */
+#include <linux/msi.h> /* for device_has_managed_irq */
 
 #include "base.h"
 #include "power/power.h"
@@ -4797,3 +4798,17 @@ int device_match_any(struct device *dev, const void *unused)
 	return 1;
 }
 EXPORT_SYMBOL_GPL(device_match_any);
+
+bool device_has_managed_msi_irq(struct device *dev)
+{
+	struct msi_desc *desc;
+
+	if (IS_ENABLED(CONFIG_GENERIC_MSI_IRQ)) {
+		for_each_msi_entry(desc, dev) {
+			if (desc->affinity && desc->affinity->is_managed)
+				return true;
+		}
+	}
+	return false;
+}
+EXPORT_SYMBOL_GPL(device_has_managed_msi_irq);
include/linux/device.h +2 -0
index 59940f1744c1..b1255524ce8b 100644
@@ -981,4 +981,6 @@ extern long sysfs_deprecated;
 #define sysfs_deprecated 0
 #endif
 
+bool device_has_managed_msi_irq(struct device *dev);
+
 #endif /* _DEVICE_H_ */
Signature · 1 line
2.31.1
#2 lore raw

Re: [PATCH V5 1/3] driver core: add device_has_managed_msi_irq

Quoted 52 lines · Ming Lei

irq vector allocation with managed affinity may be used by driver, and blk-mq needs this info for draining queue because genirq core will shutdown managed irq when all CPUs in the affinity mask are offline.

The info of using managed irq is often produced by drivers, and it is consumed by blk-mq, so different subsystems are involved in this info flow.

Address this issue by adding one helper of device_has_managed_msi_irq() which is suggested by John Garry.

---
 drivers/base/core.c    | 15 +++++++++++++++
 include/linux/device.h |  2 ++
 2 files changed, 17 insertions(+)
drivers/base/core.c +15 -0
index cadcade65825..41daf9fabdfb 100644
@@ -29,6 +29,7 @@
 #include <linux/sched/mm.h>
 #include <linux/sysfs.h>
 #include <linux/dma-map-ops.h> /* for dma_default_coherent */
+#include <linux/msi.h> /* for device_has_managed_irq */
 
 #include "base.h"
 #include "power/power.h"
@@ -4797,3 +4798,17 @@ int device_match_any(struct device *dev, const void *unused)
 	return 1;
 }
 EXPORT_SYMBOL_GPL(device_match_any);
+
+bool device_has_managed_msi_irq(struct device *dev)
+{
+	struct msi_desc *desc;
+
+	if (IS_ENABLED(CONFIG_GENERIC_MSI_IRQ)) {
+		for_each_msi_entry(desc, dev) {
+			if (desc->affinity && desc->affinity->is_managed)
+				return true;
+		}
+	}
+	return false;
+}
+EXPORT_SYMBOL_GPL(device_has_managed_msi_irq);
include/linux/device.h +0 -0
index 59940f1744c1..b1255524ce8b 100644

include/linux/device.h extern long sysfs_deprecated; · line 984

Quoted 4 lines
include/linux/device.h +1 -0
@@ -981,4 +981,6 @@ extern long sysfs_deprecated;
 #define sysfs_deprecated 0
 #endif
 
+bool device_has_managed_msi_irq(struct device *dev);

This really belongs in msi.h, not device.h, as it is very MSI specific.

Same for the .c change.

thanks,

greg k-h

#3 lore raw

Re: [PATCH V5 1/3] driver core: add device_has_managed_msi_irq

In reply to Ming Lei

drivers/base/core.c int device_match_any(struct device *dev, const void *unused) · line 4806

Quoted 5 lines · Ming Lei
drivers/base/core.c +5 -0
+bool device_has_managed_msi_irq(struct device *dev)
+{
+	struct msi_desc *desc;
+
+	if (IS_ENABLED(CONFIG_GENERIC_MSI_IRQ)) {

And inline stub for the !CONFIG_GENERIC_MSI_IRQ would seem nicer so that we don't even nee the call.

Also please add a kerneldoc comment, and as Greg said this probably doesn't belong into drivers/base and device.h but something in the irq layer. Maybe Thomas has a preference.

author
#4 lore raw

PATCH v5 2/3 blk-mq: mark if one queue map uses managed irq

In reply to Ming Lei

Retrieve this info via new added helper of device_has_managed_msi_irq, then we can decide if one hctx needs to be drained before all its CPUs become offline.

---
 block/blk-mq-pci.c                     | 1 +
 block/blk-mq-rdma.c                    | 3 +++
 block/blk-mq-virtio.c                  | 1 +
 drivers/scsi/hisi_sas/hisi_sas_v2_hw.c | 1 +
 include/linux/blk-mq.h                 | 3 ++-
 5 files changed, 8 insertions(+), 1 deletion(-)
block/blk-mq-pci.c +1 -0
index b595a94c4d16..7011854a562a 100644
@@ -37,6 +37,7 @@ int blk_mq_pci_map_queues(struct blk_mq_queue_map *qmap, struct pci_dev *pdev,
 		for_each_cpu(cpu, mask)
 			qmap->mq_map[cpu] = qmap->queue_offset + queue;
 	}
+	qmap->use_managed_irq = device_has_managed_msi_irq(&pdev->dev);
 
 	return 0;
 
block/blk-mq-rdma.c +3 -0
index 14f968e58b8f..7b10d8bd2a37 100644
@@ -36,6 +36,9 @@ int blk_mq_rdma_map_queues(struct blk_mq_queue_map *map,
 			map->mq_map[cpu] = map->queue_offset + queue;
 	}
 
+	/* So far RDMA doesn't use managed irq */
+	map->use_managed_irq = false;
+
 	return 0;
 
 fallback:
block/blk-mq-virtio.c +1 -0
index 7b8a42c35102..bea91ac5996d 100644
@@ -38,6 +38,7 @@ int blk_mq_virtio_map_queues(struct blk_mq_queue_map *qmap,
 		for_each_cpu(cpu, mask)
 			qmap->mq_map[cpu] = qmap->queue_offset + queue;
 	}
+	qmap->use_managed_irq = device_has_managed_msi_irq(&vdev->dev);
 
 	return 0;
 fallback:
drivers/scsi/hisi_sas/hisi_sas_v2_hw.c +1 -0
index b0b2361e63fe..7d7df261d346 100644
@@ -3562,6 +3562,7 @@ static int map_queues_v2_hw(struct Scsi_Host *shost)
 		for_each_cpu(cpu, mask)
 			qmap->mq_map[cpu] = qmap->queue_offset + queue;
 	}
+	qmap->use_managed_irq = true;
 
 	return 0;
 
include/linux/blk-mq.h +2 -1
index 1d18447ebebc..d54a795ec971 100644
@@ -192,7 +192,8 @@ struct blk_mq_hw_ctx {
 struct blk_mq_queue_map {
 	unsigned int *mq_map;
 	unsigned int nr_queues;
-	unsigned int queue_offset;
+	unsigned int queue_offset:31;
+	unsigned int use_managed_irq:1;
 };
 
 /**
Signature · 1 line
2.31.1
Reviewed-by on v5 2/3
#5 lore raw

Re: [PATCH V5 2/3] blk-mq: mark if one queue map uses managed irq

block/blk-mq-rdma.c int blk_mq_rdma_map_queues(struct blk_mq_queue_map *map, · line 40

Quoted 2 lines · Ming Lei
block/blk-mq-rdma.c +2 -0
+	/* So far RDMA doesn't use managed irq */
+	map->use_managed_irq = false;

It certainly did when I wrote this code. The comment should be something "sigh, someone fucked up the rdma queue mapping. No managed irqs for now".

Otherwise looks fine:

author
#6 lore raw

Re: [PATCH V5 2/3] blk-mq: mark if one queue map uses managed irq

Quoted 7 lines · Christoph Hellwig

block/blk-mq-rdma.c int blk_mq_rdma_map_queues(struct blk_mq_queue_map *map, · line 40

Quoted 2 lines · Ming Lei
block/blk-mq-rdma.c +2 -0
+	/* So far RDMA doesn't use managed irq */
+	map->use_managed_irq = false;

It certainly did when I wrote this code. The comment should be something "sigh, someone fucked up the rdma queue mapping. No managed irqs for now".

But if it rdma queue mapping uses managed irq, the issue in blk_mq_alloc_request_hctx() can be hard to fix, :-(

Thanks,
Ming

Reviewed-by on v5 2/3
#7 lore raw

Re: [PATCH V5 2/3] blk-mq: mark if one queue map uses managed irq

In reply to Ming Lei

On 21/07/2021 10:17, Ming Lei wrote:

FWIW,

include/linux/blk-mq.h struct blk_mq_hw_ctx { · line 196

Quoted 11 lines
include/linux/blk-mq.h +2 -1
index 1d18447ebebc..d54a795ec971 100644
@@ -192,7 +192,8 @@ struct blk_mq_hw_ctx {
  struct blk_mq_queue_map {
  	unsigned int *mq_map;
  	unsigned int nr_queues;
-	unsigned int queue_offset;
+	unsigned int queue_offset:31;
+	unsigned int use_managed_irq:1;

late nit: I'd be inclined to call this "drain_hw_queue" or similar, which is what it means to blk-mq

include/linux/blk-mq.h struct blk_mq_hw_ctx { · line 197

Quoted 1 line
include/linux/blk-mq.h +0 -0
  };

Thanks

author
#8 lore raw

Re: [PATCH V5 2/3] blk-mq: mark if one queue map uses managed irq

Quoted 6 lines · John Garry

On 21/07/2021 10:17, Ming Lei wrote:

FWIW,

Quoted 14 lines

include/linux/blk-mq.h struct blk_mq_hw_ctx { · line 196

Quoted 11 lines
include/linux/blk-mq.h +2 -1
index 1d18447ebebc..d54a795ec971 100644
@@ -192,7 +192,8 @@ struct blk_mq_hw_ctx {
  struct blk_mq_queue_map {
  	unsigned int *mq_map;
  	unsigned int nr_queues;
-	unsigned int queue_offset;
+	unsigned int queue_offset:31;
+	unsigned int use_managed_irq:1;

late nit: I'd be inclined to call this "drain_hw_queue" or similar, which is what it means to blk-mq

But 'drain_hw_queue' isn't straightforward when it is checked in blk_mq_alloc_request_hctx().

Thanks,
Ming

author
#9 lore raw

PATCH v5 3/3 blk-mq: don't deactivate hctx if managed irq isn't used

In reply to Ming Lei

blk-mq deactivates one hctx when the last CPU in hctx->cpumask become offline by draining all requests originated from this hctx and moving new allocation to other active hctx. This way is for avoiding inflight IO in case of managed irq because managed irq is shutdown when the last CPU in the irq's affinity becomes offline.

However, lots of drivers(nvme fc, rdma, tcp, loop, ...) don't use managed irq, so they needn't to deactivate hctx when the last CPU becomes offline. Also, some of them are the only user of blk_mq_alloc_request_hctx() which is used for connecting io queue. And their requirement is that the connect request needs to be submitted successfully via one specified hctx even though all CPUs in this hctx->cpumask have become offline.

Addressing the requirement for nvme fc/rdma/loop by allowing to allocate request from one hctx when all CPUs in this hctx are offline, since these drivers don't use managed irq.

Finally don't deactivate one hctx when it doesn't use managed irq.

---
 block/blk-mq.c | 27 +++++++++++++++++----------
 block/blk-mq.h |  8 ++++++++
 2 files changed, 25 insertions(+), 10 deletions(-)
block/blk-mq.c +17 -10
index 2c4ac51e54eb..591ab07c64d8 100644
@@ -427,6 +427,15 @@ struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
 }
 EXPORT_SYMBOL(blk_mq_alloc_request);
 
+static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
+{
+	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
+
+	if (cpu >= nr_cpu_ids)
+		cpu = cpumask_first(hctx->cpumask);
+	return cpu;
+}
+
 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
 	unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx)
 {
@@ -468,7 +477,10 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
 	data.hctx = q->queue_hw_ctx[hctx_idx];
 	if (!blk_mq_hw_queue_mapped(data.hctx))
 		goto out_queue_exit;
-	cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
+
+	WARN_ON_ONCE(blk_mq_hctx_use_managed_irq(data.hctx));
+
+	cpu = blk_mq_first_mapped_cpu(data.hctx);
 	data.ctx = __blk_mq_get_ctx(q, cpu);
 
 	if (!q->elevator)
@@ -1501,15 +1513,6 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 	hctx_unlock(hctx, srcu_idx);
 }
 
-static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
-{
-	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
-
-	if (cpu >= nr_cpu_ids)
-		cpu = cpumask_first(hctx->cpumask);
-	return cpu;
-}
-
 /*
  * It'd be great if the workqueue API had a way to pass
  * in a mask and had some smarts for more clever placement.
@@ -2556,6 +2559,10 @@ static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
 			struct blk_mq_hw_ctx, cpuhp_online);
 
+	/* hctx needn't to be deactivated in case managed irq isn't used */
+	if (!blk_mq_hctx_use_managed_irq(hctx))
+		return 0;
+
 	if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
 	    !blk_mq_last_cpu_in_hctx(cpu, hctx))
 		return 0;
block/blk-mq.h +8 -0
index d08779f77a26..7333b659d8f5 100644
@@ -119,6 +119,14 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
 	return ctx->hctxs[type];
 }
 
+static inline bool blk_mq_hctx_use_managed_irq(struct blk_mq_hw_ctx *hctx)
+{
+	if (hctx->type == HCTX_TYPE_POLL)
+		return false;
+
+	return hctx->queue->tag_set->map[hctx->type].use_managed_irq;
+}
+
 /*
  * sysfs helpers
  */
Signature · 1 line
2.31.1