Skip to contentSkip to conversation

OMAP mailbox FIFO removal

PATCH In review Andrew Davis 2 participants 19 messages Apr 10, 2024 – Jun 26, 2024 Mailbox API

lore
Share & tools
lore
#0 lore raw

v2 cover OMAP mailbox FIFO removal

Hello all,

Core of this series is the last patch removing the message FIFO from OMAP mailbox. This hurts our real-time performance. It was a legacy leftover from before the common mailbox framework anyway.

The rest of the patches are cleanups found along the way.

Thanks,
Andrew

Changes for v2:
- Use threaded irq as suggested by Hari and to fix possible "scheduling while atomic" issue
- Use oneshot irq as we do not want to enable the irq again until we clear our the messages
- Rebase on v6.9-rc3

Andrew Davis (13):
mailbox: omap: Remove unused omap_mbox_{enable,disable}_irq()
functions
mailbox: omap: Remove unused omap_mbox_request_channel() function
mailbox: omap: Move omap_mbox_irq_t into driver
mailbox: omap: Move fifo size check to point of use
mailbox: omap: Remove unneeded header omap-mailbox.h
mailbox: omap: Remove device class
mailbox: omap: Use devm_pm_runtime_enable() helper
mailbox: omap: Merge mailbox child node setup loops
mailbox: omap: Use function local struct mbox_controller
mailbox: omap: Use mbox_controller channel list directly
mailbox: omap: Remove mbox_chan_to_omap_mbox()
mailbox: omap: Reverse FIFO busy check logic
mailbox: omap: Remove kernel FIFO message queuing

 drivers/mailbox/Kconfig        |   9 -
 drivers/mailbox/omap-mailbox.c | 519 +++++++--------------------------
 include/linux/omap-mailbox.h   |  13 -
 3 files changed, 108 insertions(+), 433 deletions(-)
Signature · 1 line
2.39.2
#2 lore raw

PATCH v2 2/13 mailbox: omap: Remove unused omap_mbox_request_channel() function

This function is not used, remove this function.

---
 drivers/mailbox/omap-mailbox.c | 36 ----------------------------------
 include/linux/omap-mailbox.h   |  6 ------
 2 files changed, 42 deletions(-)
drivers/mailbox/omap-mailbox.c +0 -36
index 624a7ccc27285..8151722eef383 100644
@@ -389,42 +389,6 @@ static struct omap_mbox *omap_mbox_device_find(struct omap_mbox_device *mdev,
 	return mbox;
 }
 
-struct mbox_chan *omap_mbox_request_channel(struct mbox_client *cl,
-					    const char *chan_name)
-{
-	struct device *dev = cl->dev;
-	struct omap_mbox *mbox = NULL;
-	struct omap_mbox_device *mdev;
-	int ret;
-
-	if (!dev)
-		return ERR_PTR(-ENODEV);
-
-	if (dev->of_node) {
-		pr_err("%s: please use mbox_request_channel(), this API is supported only for OMAP non-DT usage\n",
-		       __func__);
-		return ERR_PTR(-ENODEV);
-	}
-
-	mutex_lock(&omap_mbox_devices_lock);
-	list_for_each_entry(mdev, &omap_mbox_devices, elem) {
-		mbox = omap_mbox_device_find(mdev, chan_name);
-		if (mbox)
-			break;
-	}
-	mutex_unlock(&omap_mbox_devices_lock);
-
-	if (!mbox || !mbox->chan)
-		return ERR_PTR(-ENOENT);
-
-	ret = mbox_bind_client(mbox->chan, cl);
-	if (ret)
-		return ERR_PTR(ret);
-
-	return mbox->chan;
-}
-EXPORT_SYMBOL(omap_mbox_request_channel);
-
 static struct class omap_mbox_class = { .name = "mbox", };
 
 static int omap_mbox_register(struct omap_mbox_device *mdev)
include/linux/omap-mailbox.h +0 -6
index 426a80fb32b5c..f8ddf8e814167 100644
@@ -14,10 +14,4 @@ typedef int __bitwise omap_mbox_irq_t;
 #define IRQ_TX ((__force omap_mbox_irq_t) 1)
 #define IRQ_RX ((__force omap_mbox_irq_t) 2)
 
-struct mbox_chan;
-struct mbox_client;
-
-struct mbox_chan *omap_mbox_request_channel(struct mbox_client *cl,
-					    const char *chan_name);
-
 #endif /* OMAP_MAILBOX_H */
Signature · 1 line
2.39.2
#1 lore raw

PATCH v2 1/13 mailbox: omap: Remove unused omap_mbox_{enable,disable}_irq() functions

In reply to Andrew Davis

These function are not used, remove these here.

While here, remove the leading _ from the driver internal functions that do the same thing as the functions removed.

---
 drivers/mailbox/omap-mailbox.c | 42 ++++++++--------------------------
 include/linux/omap-mailbox.h   |  3 ---
 2 files changed, 10 insertions(+), 35 deletions(-)
drivers/mailbox/omap-mailbox.c +10 -32
index c961706fe61d5..624a7ccc27285 100644
@@ -197,7 +197,7 @@ static int is_mbox_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq)
 	return (int)(enable & status & bit);
 }
 
-static void _omap_mbox_enable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq)
+static void omap_mbox_enable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq)
 {
 	u32 l;
 	struct omap_mbox_fifo *fifo = (irq == IRQ_TX) ?
@@ -210,7 +210,7 @@ static void _omap_mbox_enable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq)
 	mbox_write_reg(mbox->parent, l, irqenable);
 }
 
-static void _omap_mbox_disable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq)
+static void omap_mbox_disable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq)
 {
 	struct omap_mbox_fifo *fifo = (irq == IRQ_TX) ?
 				&mbox->tx_fifo : &mbox->rx_fifo;
@@ -227,28 +227,6 @@ static void _omap_mbox_disable_irq(struct omap_mbox *mbox, omap_mbox_irq_t irq)
 	mbox_write_reg(mbox->parent, bit, irqdisable);
 }
 
-void omap_mbox_enable_irq(struct mbox_chan *chan, omap_mbox_irq_t irq)
-{
-	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
-
-	if (WARN_ON(!mbox))
-		return;
-
-	_omap_mbox_enable_irq(mbox, irq);
-}
-EXPORT_SYMBOL(omap_mbox_enable_irq);
-
-void omap_mbox_disable_irq(struct mbox_chan *chan, omap_mbox_irq_t irq)
-{
-	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
-
-	if (WARN_ON(!mbox))
-		return;
-
-	_omap_mbox_disable_irq(mbox, irq);
-}
-EXPORT_SYMBOL(omap_mbox_disable_irq);
-
 /*
  * Message receiver(workqueue)
  */
@@ -269,7 +247,7 @@ static void mbox_rx_work(struct work_struct *work)
 		spin_lock_irq(&mq->lock);
 		if (mq->full) {
 			mq->full = false;
-			_omap_mbox_enable_irq(mq->mbox, IRQ_RX);
+			omap_mbox_enable_irq(mq->mbox, IRQ_RX);
 		}
 		spin_unlock_irq(&mq->lock);
 	}
@@ -280,7 +258,7 @@ static void mbox_rx_work(struct work_struct *work)
  */
 static void __mbox_tx_interrupt(struct omap_mbox *mbox)
 {
-	_omap_mbox_disable_irq(mbox, IRQ_TX);
+	omap_mbox_disable_irq(mbox, IRQ_TX);
 	ack_mbox_irq(mbox, IRQ_TX);
 	mbox_chan_txdone(mbox->chan, 0);
 }
@@ -293,7 +271,7 @@ static void __mbox_rx_interrupt(struct omap_mbox *mbox)
 
 	while (!mbox_fifo_empty(mbox)) {
 		if (unlikely(kfifo_avail(&mq->fifo) < sizeof(msg))) {
-			_omap_mbox_disable_irq(mbox, IRQ_RX);
+			omap_mbox_disable_irq(mbox, IRQ_RX);
 			mq->full = true;
 			goto nomem;
 		}
@@ -375,7 +353,7 @@ static int omap_mbox_startup(struct omap_mbox *mbox)
 	if (mbox->send_no_irq)
 		mbox->chan->txdone_method = TXDONE_BY_ACK;
 
-	_omap_mbox_enable_irq(mbox, IRQ_RX);
+	omap_mbox_enable_irq(mbox, IRQ_RX);
 
 	return 0;
 
@@ -386,7 +364,7 @@ static int omap_mbox_startup(struct omap_mbox *mbox)
 
 static void omap_mbox_fini(struct omap_mbox *mbox)
 {
-	_omap_mbox_disable_irq(mbox, IRQ_RX);
+	omap_mbox_disable_irq(mbox, IRQ_RX);
 	free_irq(mbox->irq, mbox);
 	flush_work(&mbox->rxq->work);
 	mbox_queue_free(mbox->rxq);
@@ -533,10 +511,10 @@ static int omap_mbox_chan_send_noirq(struct omap_mbox *mbox, u32 msg)
 	int ret = -EBUSY;
 
 	if (!mbox_fifo_full(mbox)) {
-		_omap_mbox_enable_irq(mbox, IRQ_RX);
+		omap_mbox_enable_irq(mbox, IRQ_RX);
 		mbox_fifo_write(mbox, msg);
 		ret = 0;
-		_omap_mbox_disable_irq(mbox, IRQ_RX);
+		omap_mbox_disable_irq(mbox, IRQ_RX);
 
 		/* we must read and ack the interrupt directly from here */
 		mbox_fifo_read(mbox);
@@ -556,7 +534,7 @@ static int omap_mbox_chan_send(struct omap_mbox *mbox, u32 msg)
 	}
 
 	/* always enable the interrupt */
-	_omap_mbox_enable_irq(mbox, IRQ_TX);
+	omap_mbox_enable_irq(mbox, IRQ_TX);
 	return ret;
 }
 
include/linux/omap-mailbox.h +0 -3
index 8aa984ec1f38b..426a80fb32b5c 100644
@@ -20,7 +20,4 @@ struct mbox_client;
 struct mbox_chan *omap_mbox_request_channel(struct mbox_client *cl,
 					    const char *chan_name);
 
-void omap_mbox_enable_irq(struct mbox_chan *chan, omap_mbox_irq_t irq);
-void omap_mbox_disable_irq(struct mbox_chan *chan, omap_mbox_irq_t irq);
-
 #endif /* OMAP_MAILBOX_H */
Signature · 1 line
2.39.2
#4 lore raw

PATCH v2 4/13 mailbox: omap: Move fifo size check to point of use

In reply to Andrew Davis

The mbox_kfifo_size can be changed at runtime, the sanity check on it's value should be done when it is used, not only once at init time.

---
 drivers/mailbox/omap-mailbox.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
drivers/mailbox/omap-mailbox.c +5 -5
index c083734b6954c..167348fb1b33b 100644
@@ -310,6 +310,7 @@ static struct omap_mbox_queue *mbox_queue_alloc(struct omap_mbox *mbox,
 					void (*work)(struct work_struct *))
 {
 	struct omap_mbox_queue *mq;
+	unsigned int size;
 
 	if (!work)
 		return NULL;
@@ -320,7 +321,10 @@ static struct omap_mbox_queue *mbox_queue_alloc(struct omap_mbox *mbox,
 
 	spin_lock_init(&mq->lock);
 
-	if (kfifo_alloc(&mq->fifo, mbox_kfifo_size, GFP_KERNEL))
+	/* kfifo size sanity check: alignment and minimal size */
+	size = ALIGN(mbox_kfifo_size, sizeof(u32));
+	size = max_t(unsigned int, size, sizeof(u32));
+	if (kfifo_alloc(&mq->fifo, size, GFP_KERNEL))
 		goto error;
 
 	INIT_WORK(&mq->work, work);
@@ -838,10 +842,6 @@ static int __init omap_mbox_init(void)
 	if (err)
 		return err;
 
-	/* kfifo size sanity check: alignment and minimal size */
-	mbox_kfifo_size = ALIGN(mbox_kfifo_size, sizeof(u32));
-	mbox_kfifo_size = max_t(unsigned int, mbox_kfifo_size, sizeof(u32));
-
 	err = platform_driver_register(&omap_mbox_driver);
 	if (err)
 		class_unregister(&omap_mbox_class);
Signature · 1 line
2.39.2
#6 lore raw

PATCH v2 6/13 mailbox: omap: Remove device class

In reply to Andrew Davis

The driver currently creates a new device class "mbox". Then for each mailbox adds a device to that class. This class provides no file operations provided for any userspace users of this device class. It may have been extended to be functional in our vendor tree at some point, but that is not the case anymore, nor does it matter for the upstream tree.

Remove this device class and related functions and variables. This also allows us to switch to module_platform_driver() as there is nothing left to do in module_init().

---
 drivers/mailbox/omap-mailbox.c | 89 +---------------------------------
 1 file changed, 2 insertions(+), 87 deletions(-)
drivers/mailbox/omap-mailbox.c +2 -87
index 4c673cb732ed1..ea467931faf46 100644
@@ -87,7 +87,6 @@ struct omap_mbox_device {
 	u32 intr_type;
 	struct omap_mbox **mboxes;
 	struct mbox_controller controller;
-	struct list_head elem;
 };
 
 struct omap_mbox_fifo_info {
@@ -107,7 +106,6 @@ struct omap_mbox {
 	const char		*name;
 	int			irq;
 	struct omap_mbox_queue	*rxq;
-	struct device		*dev;
 	struct omap_mbox_device *parent;
 	struct omap_mbox_fifo	tx_fifo;
 	struct omap_mbox_fifo	rx_fifo;
@@ -116,10 +114,6 @@ struct omap_mbox {
 	bool			send_no_irq;
 };
 
-/* global variables for the mailbox devices */
-static DEFINE_MUTEX(omap_mbox_devices_lock);
-static LIST_HEAD(omap_mbox_devices);
-
 static unsigned int mbox_kfifo_size = CONFIG_OMAP_MBOX_KFIFO_SIZE;
 module_param(mbox_kfifo_size, uint, S_IRUGO);
 MODULE_PARM_DESC(mbox_kfifo_size, "Size of omap's mailbox kfifo (bytes)");
@@ -395,61 +389,6 @@ static struct omap_mbox *omap_mbox_device_find(struct omap_mbox_device *mdev,
 	return mbox;
 }
 
-static struct class omap_mbox_class = { .name = "mbox", };
-
-static int omap_mbox_register(struct omap_mbox_device *mdev)
-{
-	int ret;
-	int i;
-	struct omap_mbox **mboxes;
-
-	if (!mdev || !mdev->mboxes)
-		return -EINVAL;
-
-	mboxes = mdev->mboxes;
-	for (i = 0; mboxes[i]; i++) {
-		struct omap_mbox *mbox = mboxes[i];
-
-		mbox->dev = device_create(&omap_mbox_class, mdev->dev,
-					0, mbox, "%s", mbox->name);
-		if (IS_ERR(mbox->dev)) {
-			ret = PTR_ERR(mbox->dev);
-			goto err_out;
-		}
-	}
-
-	mutex_lock(&omap_mbox_devices_lock);
-	list_add(&mdev->elem, &omap_mbox_devices);
-	mutex_unlock(&omap_mbox_devices_lock);
-
-	ret = devm_mbox_controller_register(mdev->dev, &mdev->controller);
-
-err_out:
-	if (ret) {
-		while (i--)
-			device_unregister(mboxes[i]->dev);
-	}
-	return ret;
-}
-
-static int omap_mbox_unregister(struct omap_mbox_device *mdev)
-{
-	int i;
-	struct omap_mbox **mboxes;
-
-	if (!mdev || !mdev->mboxes)
-		return -EINVAL;
-
-	mutex_lock(&omap_mbox_devices_lock);
-	list_del(&mdev->elem);
-	mutex_unlock(&omap_mbox_devices_lock);
-
-	mboxes = mdev->mboxes;
-	for (i = 0; mboxes[i]; i++)
-		device_unregister(mboxes[i]->dev);
-	return 0;
-}
-
 static int omap_mbox_chan_startup(struct mbox_chan *chan)
 {
 	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
@@ -782,7 +721,7 @@ static int omap_mbox_probe(struct platform_device *pdev)
 	mdev->controller.chans = chnls;
 	mdev->controller.num_chans = info_count;
 	mdev->controller.of_xlate = omap_mbox_of_xlate;
-	ret = omap_mbox_register(mdev);
+	ret = devm_mbox_controller_register(mdev->dev, &mdev->controller);
 	if (ret)
 		return ret;
 
@@ -809,7 +748,6 @@ static int omap_mbox_probe(struct platform_device *pdev)
 
 unregister:
 	pm_runtime_disable(mdev->dev);
-	omap_mbox_unregister(mdev);
 	return ret;
 }
 
@@ -818,7 +756,6 @@ static void omap_mbox_remove(struct platform_device *pdev)
 	struct omap_mbox_device *mdev = platform_get_drvdata(pdev);
 
 	pm_runtime_disable(mdev->dev);
-	omap_mbox_unregister(mdev);
 }
 
 static struct platform_driver omap_mbox_driver = {
@@ -830,29 +767,7 @@ static struct platform_driver omap_mbox_driver = {
 		.of_match_table = of_match_ptr(omap_mailbox_of_match),
 	},
 };
-
-static int __init omap_mbox_init(void)
-{
-	int err;
-
-	err = class_register(&omap_mbox_class);
-	if (err)
-		return err;
-
-	err = platform_driver_register(&omap_mbox_driver);
-	if (err)
-		class_unregister(&omap_mbox_class);
-
-	return err;
-}
-subsys_initcall(omap_mbox_init);
-
-static void __exit omap_mbox_exit(void)
-{
-	platform_driver_unregister(&omap_mbox_driver);
-	class_unregister(&omap_mbox_class);
-}
-module_exit(omap_mbox_exit);
+module_platform_driver(omap_mbox_driver);
 
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("omap mailbox: interrupt driven messaging");
Signature · 1 line
2.39.2
#3 lore raw

PATCH v2 3/13 mailbox: omap: Move omap_mbox_irq_t into driver

In reply to Andrew Davis

This is only used internal to the driver, move it out of the public header and into the driver file. While we are here, this is not used as a bitwise, so drop that and make it a simple enum type.

---
 drivers/mailbox/omap-mailbox.c | 5 +++++
 include/linux/omap-mailbox.h   | 4 ----
 2 files changed, 5 insertions(+), 4 deletions(-)
drivers/mailbox/omap-mailbox.c +5 -0
index 8151722eef383..c083734b6954c 100644
@@ -51,6 +51,11 @@
 #define MBOX_INTR_CFG_TYPE1		0
 #define MBOX_INTR_CFG_TYPE2		1
 
+typedef enum {
+	IRQ_TX = 1,
+	IRQ_RX = 2,
+} omap_mbox_irq_t;
+
 struct omap_mbox_fifo {
 	unsigned long msg;
 	unsigned long fifo_stat;
include/linux/omap-mailbox.h +0 -4
index f8ddf8e814167..3cc5c4ed7f5a6 100644
@@ -10,8 +10,4 @@ typedef uintptr_t mbox_msg_t;
 
 #define omap_mbox_message(data) (u32)(mbox_msg_t)(data)
 
-typedef int __bitwise omap_mbox_irq_t;
-#define IRQ_TX ((__force omap_mbox_irq_t) 1)
-#define IRQ_RX ((__force omap_mbox_irq_t) 2)
-
 #endif /* OMAP_MAILBOX_H */
Signature · 1 line
2.39.2
#7 lore raw

PATCH v2 7/13 mailbox: omap: Use devm_pm_runtime_enable() helper

In reply to Andrew Davis

Use device life-cycle managed runtime enable function to simplify probe and exit paths.

---
 drivers/mailbox/omap-mailbox.c | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)
drivers/mailbox/omap-mailbox.c +3 -15
index ea467931faf46..4f956c7b4072c 100644
@@ -726,11 +726,11 @@ static int omap_mbox_probe(struct platform_device *pdev)
 		return ret;
 
 	platform_set_drvdata(pdev, mdev);
-	pm_runtime_enable(mdev->dev);
+	devm_pm_runtime_enable(mdev->dev);
 
 	ret = pm_runtime_resume_and_get(mdev->dev);
 	if (ret < 0)
-		goto unregister;
+		return ret;
 
 	/*
 	 * just print the raw revision register, the format is not
@@ -741,26 +741,14 @@ static int omap_mbox_probe(struct platform_device *pdev)
 
 	ret = pm_runtime_put_sync(mdev->dev);
 	if (ret < 0 && ret != -ENOSYS)
-		goto unregister;
+		return ret;
 
 	devm_kfree(&pdev->dev, finfoblk);
 	return 0;
-
-unregister:
-	pm_runtime_disable(mdev->dev);
-	return ret;
-}
-
-static void omap_mbox_remove(struct platform_device *pdev)
-{
-	struct omap_mbox_device *mdev = platform_get_drvdata(pdev);
-
-	pm_runtime_disable(mdev->dev);
 }
 
 static struct platform_driver omap_mbox_driver = {
 	.probe	= omap_mbox_probe,
-	.remove_new = omap_mbox_remove,
 	.driver	= {
 		.name = "omap-mailbox",
 		.pm = &omap_mbox_pm_ops,
Signature · 1 line
2.39.2
#8 lore raw

PATCH v2 8/13 mailbox: omap: Merge mailbox child node setup loops

In reply to Andrew Davis

Currently the driver loops through all mailbox child nodes twice, once to read in data from each node, and again to make use of this data. Instead read the data and make use of it in one pass. This removes the need for several temporary data structures and reduces the complexity of this main loop in probe.

---
 drivers/mailbox/omap-mailbox.c | 119 +++++++++++++--------------------
 1 file changed, 46 insertions(+), 73 deletions(-)
drivers/mailbox/omap-mailbox.c +46 -73
index 4f956c7b4072c..17c9b9df78b1d 100644
@@ -89,19 +89,6 @@ struct omap_mbox_device {
 	struct mbox_controller controller;
 };
 
-struct omap_mbox_fifo_info {
-	int tx_id;
-	int tx_usr;
-	int tx_irq;
-
-	int rx_id;
-	int rx_usr;
-	int rx_irq;
-
-	const char *name;
-	bool send_no_irq;
-};
-
 struct omap_mbox {
 	const char		*name;
 	int			irq;
@@ -574,8 +561,7 @@ static int omap_mbox_probe(struct platform_device *pdev)
 {
 	int ret;
 	struct mbox_chan *chnls;
-	struct omap_mbox **list, *mbox, *mboxblk;
-	struct omap_mbox_fifo_info *finfo, *finfoblk;
+	struct omap_mbox **list, *mbox;
 	struct omap_mbox_device *mdev;
 	struct omap_mbox_fifo *fifo;
 	struct device_node *node = pdev->dev.of_node;
@@ -609,40 +595,6 @@ static int omap_mbox_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	finfoblk = devm_kcalloc(&pdev->dev, info_count, sizeof(*finfoblk),
-				GFP_KERNEL);
-	if (!finfoblk)
-		return -ENOMEM;
-
-	finfo = finfoblk;
-	child = NULL;
-	for (i = 0; i < info_count; i++, finfo++) {
-		child = of_get_next_available_child(node, child);
-		ret = of_property_read_u32_array(child, "ti,mbox-tx", tmp,
-						 ARRAY_SIZE(tmp));
-		if (ret)
-			return ret;
-		finfo->tx_id = tmp[0];
-		finfo->tx_irq = tmp[1];
-		finfo->tx_usr = tmp[2];
-
-		ret = of_property_read_u32_array(child, "ti,mbox-rx", tmp,
-						 ARRAY_SIZE(tmp));
-		if (ret)
-			return ret;
-		finfo->rx_id = tmp[0];
-		finfo->rx_irq = tmp[1];
-		finfo->rx_usr = tmp[2];
-
-		finfo->name = child->name;
-
-		finfo->send_no_irq = of_property_read_bool(child, "ti,mbox-send-noirq");
-
-		if (finfo->tx_id >= num_fifos || finfo->rx_id >= num_fifos ||
-		    finfo->tx_usr >= num_users || finfo->rx_usr >= num_users)
-			return -EINVAL;
-	}
-
 	mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_KERNEL);
 	if (!mdev)
 		return -ENOMEM;
@@ -667,36 +619,58 @@ static int omap_mbox_probe(struct platform_device *pdev)
 	if (!chnls)
 		return -ENOMEM;
 
-	mboxblk = devm_kcalloc(&pdev->dev, info_count, sizeof(*mbox),
-			       GFP_KERNEL);
-	if (!mboxblk)
-		return -ENOMEM;
+	child = NULL;
+	for (i = 0; i < info_count; i++) {
+		int tx_id, tx_irq, tx_usr;
+		int rx_id,         rx_usr;
+
+		mbox = devm_kzalloc(&pdev->dev, sizeof(*mbox), GFP_KERNEL);
+		if (!mbox)
+			return -ENOMEM;
+
+		child = of_get_next_available_child(node, child);
+		ret = of_property_read_u32_array(child, "ti,mbox-tx", tmp,
+						 ARRAY_SIZE(tmp));
+		if (ret)
+			return ret;
+		tx_id = tmp[0];
+		tx_irq = tmp[1];
+		tx_usr = tmp[2];
+
+		ret = of_property_read_u32_array(child, "ti,mbox-rx", tmp,
+						 ARRAY_SIZE(tmp));
+		if (ret)
+			return ret;
+		rx_id = tmp[0];
+		/* rx_irq = tmp[1]; */
+		rx_usr = tmp[2];
+
+		if (tx_id >= num_fifos || rx_id >= num_fifos ||
+		    tx_usr >= num_users || rx_usr >= num_users)
+			return -EINVAL;
 
-	mbox = mboxblk;
-	finfo = finfoblk;
-	for (i = 0; i < info_count; i++, finfo++) {
 		fifo = &mbox->tx_fifo;
-		fifo->msg = MAILBOX_MESSAGE(finfo->tx_id);
-		fifo->fifo_stat = MAILBOX_FIFOSTATUS(finfo->tx_id);
-		fifo->intr_bit = MAILBOX_IRQ_NOTFULL(finfo->tx_id);
-		fifo->irqenable = MAILBOX_IRQENABLE(intr_type, finfo->tx_usr);
-		fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, finfo->tx_usr);
-		fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, finfo->tx_usr);
+		fifo->msg = MAILBOX_MESSAGE(tx_id);
+		fifo->fifo_stat = MAILBOX_FIFOSTATUS(tx_id);
+		fifo->intr_bit = MAILBOX_IRQ_NOTFULL(tx_id);
+		fifo->irqenable = MAILBOX_IRQENABLE(intr_type, tx_usr);
+		fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, tx_usr);
+		fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, tx_usr);
 
 		fifo = &mbox->rx_fifo;
-		fifo->msg = MAILBOX_MESSAGE(finfo->rx_id);
-		fifo->msg_stat =  MAILBOX_MSGSTATUS(finfo->rx_id);
-		fifo->intr_bit = MAILBOX_IRQ_NEWMSG(finfo->rx_id);
-		fifo->irqenable = MAILBOX_IRQENABLE(intr_type, finfo->rx_usr);
-		fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, finfo->rx_usr);
-		fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, finfo->rx_usr);
-
-		mbox->send_no_irq = finfo->send_no_irq;
+		fifo->msg = MAILBOX_MESSAGE(rx_id);
+		fifo->msg_stat =  MAILBOX_MSGSTATUS(rx_id);
+		fifo->intr_bit = MAILBOX_IRQ_NEWMSG(rx_id);
+		fifo->irqenable = MAILBOX_IRQENABLE(intr_type, rx_usr);
+		fifo->irqstatus = MAILBOX_IRQSTATUS(intr_type, rx_usr);
+		fifo->irqdisable = MAILBOX_IRQDISABLE(intr_type, rx_usr);
+
+		mbox->send_no_irq = of_property_read_bool(child, "ti,mbox-send-noirq");
 		mbox->intr_type = intr_type;
 
 		mbox->parent = mdev;
-		mbox->name = finfo->name;
-		mbox->irq = platform_get_irq(pdev, finfo->tx_irq);
+		mbox->name = child->name;
+		mbox->irq = platform_get_irq(pdev, tx_irq);
 		if (mbox->irq < 0)
 			return mbox->irq;
 		mbox->chan = &chnls[i];
@@ -743,7 +717,6 @@ static int omap_mbox_probe(struct platform_device *pdev)
 	if (ret < 0 && ret != -ENOSYS)
 		return ret;
 
-	devm_kfree(&pdev->dev, finfoblk);
 	return 0;
 }
 
Signature · 1 line
2.39.2
#12 lore raw

PATCH v2 12/13 mailbox: omap: Reverse FIFO busy check logic

In reply to Andrew Davis

It is much more clear to check if the hardware FIFO is full and return EBUSY if true. This allows us to also remove one level of indention from the core of this function. It also makes the similarities between omap_mbox_chan_send_noirq() and omap_mbox_chan_send() more obvious.

---
 drivers/mailbox/omap-mailbox.c | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)
drivers/mailbox/omap-mailbox.c +16 -17
index 8e2760d2c5b0c..c5d4083125856 100644
@@ -375,34 +375,33 @@ static void omap_mbox_chan_shutdown(struct mbox_chan *chan)
 
 static int omap_mbox_chan_send_noirq(struct omap_mbox *mbox, u32 msg)
 {
-	int ret = -EBUSY;
+	if (mbox_fifo_full(mbox))
+		return -EBUSY;
 
-	if (!mbox_fifo_full(mbox)) {
-		omap_mbox_enable_irq(mbox, IRQ_RX);
-		mbox_fifo_write(mbox, msg);
-		ret = 0;
-		omap_mbox_disable_irq(mbox, IRQ_RX);
+	omap_mbox_enable_irq(mbox, IRQ_RX);
+	mbox_fifo_write(mbox, msg);
+	omap_mbox_disable_irq(mbox, IRQ_RX);
 
-		/* we must read and ack the interrupt directly from here */
-		mbox_fifo_read(mbox);
-		ack_mbox_irq(mbox, IRQ_RX);
-	}
+	/* we must read and ack the interrupt directly from here */
+	mbox_fifo_read(mbox);
+	ack_mbox_irq(mbox, IRQ_RX);
 
-	return ret;
+	return 0;
 }
 
 static int omap_mbox_chan_send(struct omap_mbox *mbox, u32 msg)
 {
-	int ret = -EBUSY;
-
-	if (!mbox_fifo_full(mbox)) {
-		mbox_fifo_write(mbox, msg);
-		ret = 0;
+	if (mbox_fifo_full(mbox)) {
+		/* always enable the interrupt */
+		omap_mbox_enable_irq(mbox, IRQ_TX);
+		return -EBUSY;
 	}
 
+	mbox_fifo_write(mbox, msg);
+
 	/* always enable the interrupt */
 	omap_mbox_enable_irq(mbox, IRQ_TX);
-	return ret;
+	return 0;
 }
 
 static int omap_mbox_chan_send_data(struct mbox_chan *chan, void *data)
Signature · 1 line
2.39.2
#5 lore raw

PATCH v2 5/13 mailbox: omap: Remove unneeded header omap-mailbox.h

In reply to Andrew Davis

The type of message sent using omap-mailbox is always u32. The definition of mbox_msg_t is uintptr_t which is wrong as that type changes based on the architecture (32bit vs 64bit). This type should have been defined as u32. Instead of making that change here, simply remove the header usage and fix the last couple users of the same in this driver.

---
 drivers/mailbox/omap-mailbox.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)
drivers/mailbox/omap-mailbox.c +2 -5
index 167348fb1b33b..4c673cb732ed1 100644
@@ -19,7 +19,6 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
-#include <linux/omap-mailbox.h>
 #include <linux/mailbox_controller.h>
 #include <linux/mailbox_client.h>
 
@@ -239,16 +238,14 @@ static void mbox_rx_work(struct work_struct *work)
 {
 	struct omap_mbox_queue *mq =
 			container_of(work, struct omap_mbox_queue, work);
-	mbox_msg_t data;
 	u32 msg;
 	int len;
 
 	while (kfifo_len(&mq->fifo) >= sizeof(msg)) {
 		len = kfifo_out(&mq->fifo, (unsigned char *)&msg, sizeof(msg));
 		WARN_ON(len != sizeof(msg));
-		data = msg;
 
-		mbox_chan_received_data(mq->mbox->chan, (void *)data);
+		mbox_chan_received_data(mq->mbox->chan, (void *)(uintptr_t)msg);
 		spin_lock_irq(&mq->lock);
 		if (mq->full) {
 			mq->full = false;
@@ -515,7 +512,7 @@ static int omap_mbox_chan_send_data(struct mbox_chan *chan, void *data)
 {
 	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
 	int ret;
-	u32 msg = omap_mbox_message(data);
+	u32 msg = (u32)(uintptr_t)(data);
 
 	if (!mbox)
 		return -EINVAL;
Signature · 1 line
2.39.2
#10 lore raw

PATCH v2 10/13 mailbox: omap: Use mbox_controller channel list directly

In reply to Andrew Davis

The driver stores a list of omap_mbox structs so it can later use it to lookup the mailbox names in of_xlate. This same information is already available in the mbox_controller passed into of_xlate. Simply use that data and remove the extra allocation and storage of the omap_mbox list.

---
 drivers/mailbox/omap-mailbox.c | 42 +++++++++-------------------------
 1 file changed, 11 insertions(+), 31 deletions(-)
drivers/mailbox/omap-mailbox.c +11 -31
index 97f59d9f9f319..8e42266cb31a5 100644
@@ -85,7 +85,6 @@ struct omap_mbox_device {
 	u32 num_users;
 	u32 num_fifos;
 	u32 intr_type;
-	struct omap_mbox **mboxes;
 };
 
 struct omap_mbox {
@@ -356,25 +355,6 @@ static void omap_mbox_fini(struct omap_mbox *mbox)
 	mbox_queue_free(mbox->rxq);
 }
 
-static struct omap_mbox *omap_mbox_device_find(struct omap_mbox_device *mdev,
-					       const char *mbox_name)
-{
-	struct omap_mbox *_mbox, *mbox = NULL;
-	struct omap_mbox **mboxes = mdev->mboxes;
-	int i;
-
-	if (!mboxes)
-		return NULL;
-
-	for (i = 0; (_mbox = mboxes[i]); i++) {
-		if (!strcmp(_mbox->name, mbox_name)) {
-			mbox = _mbox;
-			break;
-		}
-	}
-	return mbox;
-}
-
 static int omap_mbox_chan_startup(struct mbox_chan *chan)
 {
 	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
@@ -539,6 +519,7 @@ static struct mbox_chan *omap_mbox_of_xlate(struct mbox_controller *controller,
 	struct device_node *node;
 	struct omap_mbox_device *mdev;
 	struct omap_mbox *mbox;
+	int i;
 
 	mdev = dev_get_drvdata(controller->dev);
 	if (WARN_ON(!mdev))
@@ -551,16 +532,23 @@ static struct mbox_chan *omap_mbox_of_xlate(struct mbox_controller *controller,
 		return ERR_PTR(-ENODEV);
 	}
 
-	mbox = omap_mbox_device_find(mdev, node->name);
+	for (i = 0; i < controller->num_chans; i++) {
+		mbox = controller->chans[i].con_priv;
+		if (!strcmp(mbox->name, node->name)) {
+			of_node_put(node);
+			return &controller->chans[i];
+		}
+	}
+
 	of_node_put(node);
-	return mbox ? mbox->chan : ERR_PTR(-ENOENT);
+	return ERR_PTR(-ENOENT);
 }
 
 static int omap_mbox_probe(struct platform_device *pdev)
 {
 	int ret;
 	struct mbox_chan *chnls;
-	struct omap_mbox **list, *mbox;
+	struct omap_mbox *mbox;
 	struct omap_mbox_device *mdev;
 	struct omap_mbox_fifo *fifo;
 	struct device_node *node = pdev->dev.of_node;
@@ -608,12 +596,6 @@ static int omap_mbox_probe(struct platform_device *pdev)
 	if (!mdev->irq_ctx)
 		return -ENOMEM;
 
-	/* allocate one extra for marking end of list */
-	list = devm_kcalloc(&pdev->dev, info_count + 1, sizeof(*list),
-			    GFP_KERNEL);
-	if (!list)
-		return -ENOMEM;
-
 	chnls = devm_kcalloc(&pdev->dev, info_count + 1, sizeof(*chnls),
 			     GFP_KERNEL);
 	if (!chnls)
@@ -675,7 +657,6 @@ static int omap_mbox_probe(struct platform_device *pdev)
 			return mbox->irq;
 		mbox->chan = &chnls[i];
 		chnls[i].con_priv = mbox;
-		list[i] = mbox++;
 	}
 
 	mutex_init(&mdev->cfg_lock);
@@ -683,7 +664,6 @@ static int omap_mbox_probe(struct platform_device *pdev)
 	mdev->num_users = num_users;
 	mdev->num_fifos = num_fifos;
 	mdev->intr_type = intr_type;
-	mdev->mboxes = list;
 
 	controller = devm_kzalloc(&pdev->dev, sizeof(*controller), GFP_KERNEL);
 	if (!controller)
Signature · 1 line
2.39.2
#13 lore raw

PATCH v2 13/13 mailbox: omap: Remove kernel FIFO message queuing

In reply to Andrew Davis

The kernel FIFO queue has a couple issues. The biggest issue is that it causes extra latency in a path that can be used in real-time tasks, such as communication with real-time remote processors.

The whole FIFO idea itself looks to be a leftover from before the unified mailbox framework. The current mailbox framework expects mbox_chan_received_data() to be called with data immediately as it arrives. Remove the FIFO and pass the messages to the mailbox framework directly as part of a threaded IRQ handler.

---
 drivers/mailbox/Kconfig        |   9 ---
 drivers/mailbox/omap-mailbox.c | 107 ++-------------------------------
 2 files changed, 5 insertions(+), 111 deletions(-)
drivers/mailbox/Kconfig +0 -9
index 42940108a1874..78e4c74fbe5c2 100644
@@ -68,15 +68,6 @@ config OMAP2PLUS_MBOX
 	  OMAP2/3; or IPU, IVA HD and DSP in OMAP4/5. Say Y here if you
 	  want to use OMAP2+ Mailbox framework support.
 
-config OMAP_MBOX_KFIFO_SIZE
-	int "Mailbox kfifo default buffer size (bytes)"
-	depends on OMAP2PLUS_MBOX
-	default 256
-	help
-	  Specify the default size of mailbox's kfifo buffers (bytes).
-	  This can also be changed at runtime (via the mbox_kfifo_size
-	  module parameter).
-
 config ROCKCHIP_MBOX
 	bool "Rockchip Soc Integrated Mailbox Support"
 	depends on ARCH_ROCKCHIP || COMPILE_TEST
drivers/mailbox/omap-mailbox.c +5 -102

168 lines not on this page · raw message

Signature · 1 line
2.39.2
#11 lore raw

PATCH v2 11/13 mailbox: omap: Remove mbox_chan_to_omap_mbox()

In reply to Andrew Davis

This function only checks if mbox_chan *chan is not NULL, but that cannot be the case and if it was returning NULL which is not later checked doesn't save us from this. The second check for chan->con_priv is completely redundant as if it was NULL we would return NULL just the same. Simply dereference con_priv directly and remove this function.

---
 drivers/mailbox/omap-mailbox.c | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)
drivers/mailbox/omap-mailbox.c +3 -11
index 8e42266cb31a5..8e2760d2c5b0c 100644
@@ -103,14 +103,6 @@ static unsigned int mbox_kfifo_size = CONFIG_OMAP_MBOX_KFIFO_SIZE;
 module_param(mbox_kfifo_size, uint, S_IRUGO);
 MODULE_PARM_DESC(mbox_kfifo_size, "Size of omap's mailbox kfifo (bytes)");
 
-static struct omap_mbox *mbox_chan_to_omap_mbox(struct mbox_chan *chan)
-{
-	if (!chan || !chan->con_priv)
-		return NULL;
-
-	return (struct omap_mbox *)chan->con_priv;
-}
-
 static inline
 unsigned int mbox_read_reg(struct omap_mbox_device *mdev, size_t ofs)
 {
@@ -357,7 +349,7 @@ static void omap_mbox_fini(struct omap_mbox *mbox)
 
 static int omap_mbox_chan_startup(struct mbox_chan *chan)
 {
-	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
+	struct omap_mbox *mbox = chan->con_priv;
 	struct omap_mbox_device *mdev = mbox->parent;
 	int ret = 0;
 
@@ -372,7 +364,7 @@ static int omap_mbox_chan_startup(struct mbox_chan *chan)
 
 static void omap_mbox_chan_shutdown(struct mbox_chan *chan)
 {
-	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
+	struct omap_mbox *mbox = chan->con_priv;
 	struct omap_mbox_device *mdev = mbox->parent;
 
 	mutex_lock(&mdev->cfg_lock);
@@ -415,7 +407,7 @@ static int omap_mbox_chan_send(struct omap_mbox *mbox, u32 msg)
 
 static int omap_mbox_chan_send_data(struct mbox_chan *chan, void *data)
 {
-	struct omap_mbox *mbox = mbox_chan_to_omap_mbox(chan);
+	struct omap_mbox *mbox = chan->con_priv;
 	int ret;
 	u32 msg = (u32)(uintptr_t)(data);
 
Signature · 1 line
2.39.2
#9 lore raw

PATCH v2 9/13 mailbox: omap: Use function local struct mbox_controller

In reply to Andrew Davis

The mbox_controller struct is only needed in the probe function. Make it a local variable instead of storing a copy in omap_mbox_device to simplify that struct.

---
 drivers/mailbox/omap-mailbox.c | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)
drivers/mailbox/omap-mailbox.c +12 -9
index 17c9b9df78b1d..97f59d9f9f319 100644
@@ -86,7 +86,6 @@ struct omap_mbox_device {
 	u32 num_fifos;
 	u32 intr_type;
 	struct omap_mbox **mboxes;
-	struct mbox_controller controller;
 };
 
 struct omap_mbox {
@@ -541,7 +540,7 @@ static struct mbox_chan *omap_mbox_of_xlate(struct mbox_controller *controller,
 	struct omap_mbox_device *mdev;
 	struct omap_mbox *mbox;
 
-	mdev = container_of(controller, struct omap_mbox_device, controller);
+	mdev = dev_get_drvdata(controller->dev);
 	if (WARN_ON(!mdev))
 		return ERR_PTR(-EINVAL);
 
@@ -567,6 +566,7 @@ static int omap_mbox_probe(struct platform_device *pdev)
 	struct device_node *node = pdev->dev.of_node;
 	struct device_node *child;
 	const struct omap_mbox_match_data *match_data;
+	struct mbox_controller *controller;
 	u32 intr_type, info_count;
 	u32 num_users, num_fifos;
 	u32 tmp[3];
@@ -685,17 +685,20 @@ static int omap_mbox_probe(struct platform_device *pdev)
 	mdev->intr_type = intr_type;
 	mdev->mboxes = list;
 
+	controller = devm_kzalloc(&pdev->dev, sizeof(*controller), GFP_KERNEL);
+	if (!controller)
+		return -ENOMEM;
 	/*
 	 * OMAP/K3 Mailbox IP does not have a Tx-Done IRQ, but rather a Tx-Ready
 	 * IRQ and is needed to run the Tx state machine
 	 */
-	mdev->controller.txdone_irq = true;
-	mdev->controller.dev = mdev->dev;
-	mdev->controller.ops = &omap_mbox_chan_ops;
-	mdev->controller.chans = chnls;
-	mdev->controller.num_chans = info_count;
-	mdev->controller.of_xlate = omap_mbox_of_xlate;
-	ret = devm_mbox_controller_register(mdev->dev, &mdev->controller);
+	controller->txdone_irq = true;
+	controller->dev = mdev->dev;
+	controller->ops = &omap_mbox_chan_ops;
+	controller->chans = chnls;
+	controller->num_chans = info_count;
+	controller->of_xlate = omap_mbox_of_xlate;
+	ret = devm_mbox_controller_register(mdev->dev, controller);
 	if (ret)
 		return ret;
 
Signature · 1 line
2.39.2
#14 lore raw
In reply to Andrew Davis

Hello Andrew,

Quoted 3 lines · Andrew Davis

Changes for v2:
- Use threaded irq as suggested by Hari and to fix possible "scheduling while atomic" issue

sorry for beeing late, I noticed this already got merged.

I was wondering what the reason was for ending up with the threaded irq.

In your v1 thread your final conclusion appeared to be

Quoted 1 line

So for now I just kept this using the regular IRQ context as before.

We looked into this some time ago, and noticed that the IRQ approach caused problems in the virtio/rpmsg code. I'd like to understand if your change was for the same reason, or something else we missed before.

Regards,

Dominic

#15 lore raw
Quoted 9 lines · Dominic Rath

Hello Andrew,

Quoted 3 lines · Andrew Davis

Changes for v2:
  - Use threaded irq as suggested by Hari and to fix possible "scheduling while atomic" issue

sorry for beeing late, I noticed this already got merged.

Quoted 9 lines

I was wondering what the reason was for ending up with the threaded irq.

In your v1 thread your final conclusion appeared to be

Quoted 1 line

So for now I just kept this using the regular IRQ context as before.

We looked into this some time ago, and noticed that the IRQ approach caused problems in the virtio/rpmsg code. I'd like to understand if your change was for the same reason, or something else we missed before.

It is most likely the same reason. Seems despite its name, rproc_vq_interrupt() cannot be called from an IRQ/atomic context. As the following backtrace shows, that function calls down into functions which are not IRQ safe. So we needed to keep it threaded:

[    5.389374] BUG: scheduling while atomic: (udev-worker)/232/0x00010002
[    5.395917] Modules linked in: videobuf2_dma_contig videobuf2_memops videobuf2_v4l2 phy_j721e_wiz display_connector omap_mailbox(+) videodev tps6594_i2c(+) videobuf2_common phy_can_transceiver at24 cd6
[    5.433562] CPU: 0 PID: 232 Comm: (udev-worker) Not tainted 6.10.0-rc1-next-20240528-dirty #10
[    5.442158] Hardware name: Texas Instruments AM69 SK (DT)
[    5.447540] Call trace:
[    5.449976]  dump_backtrace+0x94/0xec
[    5.453640]  show_stack+0x18/0x24
[    5.456944]  dump_stack_lvl+0x78/0x90
[    5.460598]  dump_stack+0x18/0x24
[    5.463900]  __schedule_bug+0x50/0x68
[    5.467552]  __schedule+0x80c/0xb0c
[    5.471029]  schedule+0x34/0x104
[    5.474243]  schedule_preempt_disabled+0x24/0x40
[    5.478845]  rwsem_down_write_slowpath+0x31c/0x56c
[    5.483622]  down_write+0x90/0x94
[    5.486924]  kernfs_add_one+0x3c/0x148
[    5.490661]  kernfs_create_dir_ns+0x50/0x94
[    5.494830]  sysfs_create_dir_ns+0x70/0x10c
[    5.498999]  kobject_add_internal+0x98/0x26c
[    5.503254]  kobject_add+0x9c/0x10c
[    5.506729]  device_add+0xc0/0x790
[    5.510120]  rpmsg_register_device_override+0x10c/0x1c0
[    5.515333]  rpmsg_register_device+0x14/0x20
[    5.519590]  virtio_rpmsg_create_channel+0xb0/0x104
[    5.524452]  rpmsg_create_channel+0x28/0x60
[    5.528622]  rpmsg_ns_cb+0x100/0x1dc
[    5.532185]  rpmsg_recv_done+0x114/0x2e4
[    5.536094]  vring_interrupt+0x68/0xa4
[    5.539833]  rproc_vq_interrupt+0x2c/0x48
[    5.543830]  k3_r5_rproc_mbox_callback+0x84/0x90 [ti_k3_r5_remoteproc]
[    5.550348]  mbox_chan_received_data+0x1c/0x2c
[    5.554779]  mbox_interrupt+0xa0/0x17c [omap_mailbox]
[    5.559820]  __handle_irq_event_percpu+0x48/0x13c
[    5.564511]  handle_irq_event+0x4c/0xac

Andrew

Quoted 3 lines

Regards,

Dominic

#16 lore raw
Quoted 6 lines · Andrew Davis
Quoted 2 lines

We looked into this some time ago, and noticed that the IRQ approach caused problems in the virtio/rpmsg code. I'd like to understand if your change was for the same reason, or something else we missed before.

It is most likely the same reason. Seems despite its name, rproc_vq_interrupt() cannot be called from an IRQ/atomic context. As the following backtrace shows, that function calls down into functions which are not IRQ safe. So we needed to keep it threaded:

Thanks for confirming. This is exactly what we've been seeing.

Regards,

Dominic

#17 lore raw
In reply to Andrew Davis
Quoted 33 lines · Andrew Davis
Quoted 2 lines

We looked into this some time ago, and noticed that the IRQ approach caused problems in the virtio/rpmsg code. I'd like to understand if your change was for the same reason, or something else we missed before.

It is most likely the same reason. Seems despite its name, rproc_vq_interrupt() cannot be called from an IRQ/atomic context. As the following backtrace shows, that function calls down into functions which are not IRQ safe. So we needed to keep it threaded:

[    5.389374] BUG: scheduling while atomic: (udev-worker)/232/0x00010002
[    5.395917] Modules linked in: videobuf2_dma_contig videobuf2_memops videobuf2_v4l2 phy_j721e_wiz display_connector omap_mailbox(+) videodev tps6594_i2c(+) videobuf2_common phy_can_transceiver at24 cd6 [    5.433562] CPU: 0 PID: 232 Comm: (udev-worker) Not tainted 6.10.0-rc1-next-20240528-dirty #10
[    5.442158] Hardware name: Texas Instruments AM69 SK (DT)
[    5.447540] Call trace:
[    5.449976]  dump_backtrace+0x94/0xec
[    5.453640]  show_stack+0x18/0x24
[    5.456944]  dump_stack_lvl+0x78/0x90
[    5.460598]  dump_stack+0x18/0x24
[    5.463900]  __schedule_bug+0x50/0x68
[    5.467552]  __schedule+0x80c/0xb0c
[    5.471029]  schedule+0x34/0x104
[    5.474243]  schedule_preempt_disabled+0x24/0x40
[    5.478845]  rwsem_down_write_slowpath+0x31c/0x56c
[    5.483622]  down_write+0x90/0x94
[    5.486924]  kernfs_add_one+0x3c/0x148
[    5.490661]  kernfs_create_dir_ns+0x50/0x94
[    5.494830]  sysfs_create_dir_ns+0x70/0x10c
[    5.498999]  kobject_add_internal+0x98/0x26c
[    5.503254]  kobject_add+0x9c/0x10c
[    5.506729]  device_add+0xc0/0x790
[    5.510120]  rpmsg_register_device_override+0x10c/0x1c0
[    5.515333]  rpmsg_register_device+0x14/0x20
[    5.519590]  virtio_rpmsg_create_channel+0xb0/0x104
[    5.524452]  rpmsg_create_channel+0x28/0x60
[    5.528622]  rpmsg_ns_cb+0x100/0x1dc

Quoted 9 lines

[    5.532185]  rpmsg_recv_done+0x114/0x2e4
[    5.536094]  vring_interrupt+0x68/0xa4
[    5.539833]  rproc_vq_interrupt+0x2c/0x48
[    5.543830]  k3_r5_rproc_mbox_callback+0x84/0x90 [ti_k3_r5_remoteproc] [    5.550348]  mbox_chan_received_data+0x1c/0x2c
[    5.554779]  mbox_interrupt+0xa0/0x17c [omap_mailbox]
[    5.559820]  __handle_irq_event_percpu+0x48/0x13c
[    5.564511]  handle_irq_event+0x4c/0xac

I looked into this a bit more closely, together with the colleague who implemented our internal workaround, and we came up with one more concern:

Have you considered that this synchronous path from the (threaded) IRQ draining the mailbox down to the (potentially blocking) rpmsg_* calls might let the hardware mailbox grow full?

From what I remember the vring (?) has room for 512 messages, but the hardware mailbox on e.g. the AM64x can only handle four messages. At least with the current implementation on TI's MCU+ SDK running on the R5f that could cause the R5f to block, waiting for room in the hardware mailbox, while there are plenty of vring buffers available.

Best Regards,

Dominic

#18 lore raw
Quoted 44 lines · Dominic Rath
Quoted 33 lines · Andrew Davis
Quoted 2 lines

We looked into this some time ago, and noticed that the IRQ approach caused problems in the virtio/rpmsg code. I'd like to understand if your change was for the same reason, or something else we missed before.

It is most likely the same reason. Seems despite its name, rproc_vq_interrupt() cannot be called from an IRQ/atomic context. As the following backtrace shows, that function calls down into functions which are not IRQ safe. So we needed to keep it threaded:

[    5.389374] BUG: scheduling while atomic: (udev-worker)/232/0x00010002
[    5.395917] Modules linked in: videobuf2_dma_contig videobuf2_memops videobuf2_v4l2 phy_j721e_wiz display_connector omap_mailbox(+) videodev tps6594_i2c(+) videobuf2_common phy_can_transceiver at24 cd6 [    5.433562] CPU: 0 PID: 232 Comm: (udev-worker) Not tainted 6.10.0-rc1-next-20240528-dirty #10
[    5.442158] Hardware name: Texas Instruments AM69 SK (DT)
[    5.447540] Call trace:
[    5.449976]  dump_backtrace+0x94/0xec
[    5.453640]  show_stack+0x18/0x24
[    5.456944]  dump_stack_lvl+0x78/0x90
[    5.460598]  dump_stack+0x18/0x24
[    5.463900]  __schedule_bug+0x50/0x68
[    5.467552]  __schedule+0x80c/0xb0c
[    5.471029]  schedule+0x34/0x104
[    5.474243]  schedule_preempt_disabled+0x24/0x40
[    5.478845]  rwsem_down_write_slowpath+0x31c/0x56c
[    5.483622]  down_write+0x90/0x94
[    5.486924]  kernfs_add_one+0x3c/0x148
[    5.490661]  kernfs_create_dir_ns+0x50/0x94
[    5.494830]  sysfs_create_dir_ns+0x70/0x10c
[    5.498999]  kobject_add_internal+0x98/0x26c
[    5.503254]  kobject_add+0x9c/0x10c
[    5.506729]  device_add+0xc0/0x790
[    5.510120]  rpmsg_register_device_override+0x10c/0x1c0
[    5.515333]  rpmsg_register_device+0x14/0x20
[    5.519590]  virtio_rpmsg_create_channel+0xb0/0x104
[    5.524452]  rpmsg_create_channel+0x28/0x60
[    5.528622]  rpmsg_ns_cb+0x100/0x1dc

Quoted 9 lines

[    5.532185]  rpmsg_recv_done+0x114/0x2e4
[    5.536094]  vring_interrupt+0x68/0xa4
[    5.539833]  rproc_vq_interrupt+0x2c/0x48
[    5.543830]  k3_r5_rproc_mbox_callback+0x84/0x90 [ti_k3_r5_remoteproc] [    5.550348]  mbox_chan_received_data+0x1c/0x2c
[    5.554779]  mbox_interrupt+0xa0/0x17c [omap_mailbox]
[    5.559820]  __handle_irq_event_percpu+0x48/0x13c
[    5.564511]  handle_irq_event+0x4c/0xac

Quoted 6 lines

I looked into this a bit more closely, together with the colleague who implemented our internal workaround, and we came up with one more concern:

Have you considered that this synchronous path from the (threaded) IRQ draining the mailbox down to the (potentially blocking) rpmsg_* calls might let the hardware mailbox grow full?

From what I remember the vring (?) has room for 512 messages, but the hardware mailbox on e.g. the AM64x can only handle four messages. At least with the current implementation on TI's MCU+ SDK running on the R5f that could cause the R5f to block, waiting for room in the hardware mailbox, while there are plenty of vring buffers available.

We would like to switch back to the non-threaded handler at some point. As you mention doing this in a threaded way increase the risk of the hardware message queue filling and blocking the remote side. (Plus the threaded handling can add latency to the message handling which should be avoided for real-time reasons)

The fix might be to have rpmsg_recv_done() callback simply queue the message and then schedule another worker to actually do the next stage processing. That way complex actions on messages do not block vring_interrupt() which should be treated like an atomic context call.

Anyway for now, I'd expect the much faster host core (2xA53 @ 1GHZ in AM64) to be able to outpace the R5s and keep the mailbox drained. Are you actually running into this issue or is the concern based on ensuring RT performance(not blocking on mailbox queue filled) on the R5 side?

Andrew

Quoted 3 lines

Best Regards,

Dominic