Skip to contentSkip to conversation

PCI/DOE: Remove asynchronous task support

PATCH Superseded [email protected] 3 participants 9 messages Nov 22, 2022 – Nov 24, 2022

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover PCI/DOE: Remove asynchronous task support

From: Ira Weiny <[email protected]>

Changes from V1
Address comments around locking
Add patch to remove unneeded pci_doe_flush_mb() call

The asynchronous task support in the PCI DOE mailboxes has become a maintenance burden.

Remove the code associated with that support.

Ira Weiny (2):
PCI/DOE: Remove the pci_doe_flush_mb() call
PCI/DOE: Remove asynchronous task support

 drivers/cxl/core/pci.c  |  16 +-----
 drivers/pci/doe.c       | 123 +++++++++-------------------------------
 include/linux/pci-doe.h |  10 +---
 3 files changed, 32 insertions(+), 117 deletions(-)

base-commit: b6e7fdfd6f6a8bf88fcdb4a45da52c42ba238c25

Signature · 1 line
2.37.2
#1 lore raw

PATCH v2 1/2 PCI/DOE: Remove the pci_doe_flush_mb() call

From: Ira Weiny <[email protected]>

Each struct doe_mb is managed as part of the PCI device. They can't go away as long as the PCI device exists. pci_doe_flush_mb() was set up to flush the workqueue and prevent any further submissions to the mailboxes when the PCI device goes away. Unfortunately, this was fundamentally flawed. There was no guarantee that a struct doe_mb remained after pci_doe_flush_mb() returned. Therefore, the doe_mb state could be invalid when those threads waiting on the workqueue were flushed.

Fortunately the current code is safe because all callers make a synchronous call to pci_doe_submit_task() and maintain a reference on the PCI device.

For these reasons, pci_doe_flush_mb() will never be called while tasks are being processed and there is no use for it.

Remove the dead code around pci_doe_flush_mb().

---
 drivers/pci/doe.c | 48 ++++-------------------------------------------
 1 file changed, 4 insertions(+), 44 deletions(-)
drivers/pci/doe.c +4 -44
index e402f05068a5..260313e9052e 100644
@@ -24,10 +24,9 @@
 
 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
 #define PCI_DOE_TIMEOUT HZ
-#define PCI_DOE_POLL_INTERVAL	(PCI_DOE_TIMEOUT / 128)
+#define PCI_DOE_POLL_INTERVAL	8
 
-#define PCI_DOE_FLAG_CANCEL	0
-#define PCI_DOE_FLAG_DEAD	1
+#define PCI_DOE_FLAG_DEAD	0
 
 /**
  * struct pci_doe_mb - State for a single DOE mailbox
@@ -53,15 +52,6 @@ struct pci_doe_mb {
 	unsigned long flags;
 };
 
-static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
-{
-	if (wait_event_timeout(doe_mb->wq,
-			       test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
-			       timeout))
-		return -EIO;
-	return 0;
-}
-
 static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
 {
 	struct pci_dev *pdev = doe_mb->pdev;
@@ -82,12 +72,9 @@ static int pci_doe_abort(struct pci_doe_mb *doe_mb)
 	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
 
 	do {
-		int rc;
 		u32 val;
 
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc)
-			return rc;
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
 
 		/* Abort success! */
@@ -278,11 +265,7 @@ static void doe_statemachine_work(struct work_struct *work)
 			signal_task_abort(task, -EIO);
 			return;
 		}
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc) {
-			signal_task_abort(task, rc);
-			return;
-		}
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		goto retry_resp;
 	}
 
@@ -383,21 +366,6 @@ static void pci_doe_destroy_workqueue(void *mb)
 	destroy_workqueue(doe_mb->work_queue);
 }
 
-static void pci_doe_flush_mb(void *mb)
-{
-	struct pci_doe_mb *doe_mb = mb;
-
-	/* Stop all pending work items from starting */
-	set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
-
-	/* Cancel an in progress work item, if necessary */
-	set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
-	wake_up(&doe_mb->wq);
-
-	/* Flush all work items */
-	flush_workqueue(doe_mb->work_queue);
-}
-
 /**
  * pcim_doe_create_mb() - Create a DOE mailbox object
  *
@@ -450,14 +418,6 @@ struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
 		return ERR_PTR(rc);
 	}
 
-	/*
-	 * The state machine and the mailbox should be in sync now;
-	 * Set up mailbox flush prior to using the mailbox to query protocols.
-	 */
-	rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
-	if (rc)
-		return ERR_PTR(rc);
-
 	rc = pci_doe_cache_protocols(doe_mb);
 	if (rc) {
 		pci_err(pdev, "[%x] failed to cache protocols : %d\n",
Signature · 1 line
2.37.2
#2 lore raw

Re: [PATCH V2 1/2] PCI/DOE: Remove the pci_doe_flush_mb() call

Quoted 14 lines · On Tue, 22 Nov 2022 07:53:23 -0800 [email protected] wrote:

From: Ira Weiny <[email protected]>

Each struct doe_mb is managed as part of the PCI device. They can't go away as long as the PCI device exists. pci_doe_flush_mb() was set up to flush the workqueue and prevent any further submissions to the mailboxes when the PCI device goes away. Unfortunately, this was fundamentally flawed. There was no guarantee that a struct doe_mb remained after pci_doe_flush_mb() returned. Therefore, the doe_mb state could be invalid when those threads waiting on the workqueue were flushed.

Fortunately the current code is safe because all callers make a synchronous call to pci_doe_submit_task() and maintain a reference on the PCI device.

Quoted 6 lines

For these reasons, pci_doe_flush_mb() will never be called while tasks are being processed and there is no use for it.

Remove the dead code around pci_doe_flush_mb().

Looks fine I think, though one question inline.

Quoted 9 lines
---
 drivers/pci/doe.c | 48 ++++-------------------------------------------
 1 file changed, 4 insertions(+), 44 deletions(-)
drivers/pci/doe.c +0 -0
index e402f05068a5..260313e9052e 100644

drivers/pci/doe.c line 27

Quoted 5 lines
drivers/pci/doe.c +1 -1
@@ -24,10 +24,9 @@
 
 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
 #define PCI_DOE_TIMEOUT HZ
-#define PCI_DOE_POLL_INTERVAL	(PCI_DOE_TIMEOUT / 128)
+#define PCI_DOE_POLL_INTERVAL	8

Why this change?

Quoted 87 lines
drivers/pci/doe.c +3 -43
-#define PCI_DOE_FLAG_CANCEL	0
-#define PCI_DOE_FLAG_DEAD	1
+#define PCI_DOE_FLAG_DEAD	0
 
 /**
  * struct pci_doe_mb - State for a single DOE mailbox
@@ -53,15 +52,6 @@ struct pci_doe_mb {
 	unsigned long flags;
 };
 
-static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
-{
-	if (wait_event_timeout(doe_mb->wq,
-			       test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
-			       timeout))
-		return -EIO;
-	return 0;
-}
-
 static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
 {
 	struct pci_dev *pdev = doe_mb->pdev;
@@ -82,12 +72,9 @@ static int pci_doe_abort(struct pci_doe_mb *doe_mb)
 	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
 
 	do {
-		int rc;
 		u32 val;
 
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc)
-			return rc;
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
 
 		/* Abort success! */
@@ -278,11 +265,7 @@ static void doe_statemachine_work(struct work_struct *work)
 			signal_task_abort(task, -EIO);
 			return;
 		}
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc) {
-			signal_task_abort(task, rc);
-			return;
-		}
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		goto retry_resp;
 	}
 
@@ -383,21 +366,6 @@ static void pci_doe_destroy_workqueue(void *mb)
 	destroy_workqueue(doe_mb->work_queue);
 }
 
-static void pci_doe_flush_mb(void *mb)
-{
-	struct pci_doe_mb *doe_mb = mb;
-
-	/* Stop all pending work items from starting */
-	set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
-
-	/* Cancel an in progress work item, if necessary */
-	set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
-	wake_up(&doe_mb->wq);
-
-	/* Flush all work items */
-	flush_workqueue(doe_mb->work_queue);
-}
-
 /**
  * pcim_doe_create_mb() - Create a DOE mailbox object
  *
@@ -450,14 +418,6 @@ struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
 		return ERR_PTR(rc);
 	}
 
-	/*
-	 * The state machine and the mailbox should be in sync now;
-	 * Set up mailbox flush prior to using the mailbox to query protocols.
-	 */
-	rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
-	if (rc)
-		return ERR_PTR(rc);
-
 	rc = pci_doe_cache_protocols(doe_mb);
 	if (rc) {
 		pci_err(pdev, "[%x] failed to cache protocols : %d\n",
author
#3 lore raw

Re: [PATCH V2 1/2] PCI/DOE: Remove the pci_doe_flush_mb() call

Quoted 36 lines · Jonathan Cameron
Quoted 14 lines · On Tue, 22 Nov 2022 07:53:23 -0800 [email protected] wrote:

From: Ira Weiny <[email protected]>

Each struct doe_mb is managed as part of the PCI device. They can't go away as long as the PCI device exists. pci_doe_flush_mb() was set up to flush the workqueue and prevent any further submissions to the mailboxes when the PCI device goes away. Unfortunately, this was fundamentally flawed. There was no guarantee that a struct doe_mb remained after pci_doe_flush_mb() returned. Therefore, the doe_mb state could be invalid when those threads waiting on the workqueue were flushed.

Fortunately the current code is safe because all callers make a synchronous call to pci_doe_submit_task() and maintain a reference on the PCI device.

Quoted 6 lines

For these reasons, pci_doe_flush_mb() will never be called while tasks are being processed and there is no use for it.

Remove the dead code around pci_doe_flush_mb().

Looks fine I think, though one question inline.

Quoted 10 lines
---
 drivers/pci/doe.c | 48 ++++-------------------------------------------
 1 file changed, 4 insertions(+), 44 deletions(-)
drivers/pci/doe.c +0 -0
index e402f05068a5..260313e9052e 100644
@@ -24,10 +24,9 @@
 
Quoted 6 lines

drivers/pci/doe.c line 27

Quoted 4 lines
drivers/pci/doe.c +1 -1
 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
 #define PCI_DOE_TIMEOUT HZ
-#define PCI_DOE_POLL_INTERVAL	(PCI_DOE_TIMEOUT / 128)
+#define PCI_DOE_POLL_INTERVAL	8

Why this change?

msleep_interruptible() takes a millisecond value and wait_event_timeout() takes jiffies. 1/128 of a second is ~8ms.

While for most configs (HZ == 1000) the value does not change. I don't believe this would be true for all configs. Thus a more explicit define.

I'll add a comment.

Ira

Quoted 89 lines
Quoted 88 lines
drivers/pci/doe.c +3 -43
-#define PCI_DOE_FLAG_CANCEL	0
-#define PCI_DOE_FLAG_DEAD	1
+#define PCI_DOE_FLAG_DEAD	0
 
 /**
  * struct pci_doe_mb - State for a single DOE mailbox
@@ -53,15 +52,6 @@ struct pci_doe_mb {
 	unsigned long flags;
 };
 
-static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
-{
-	if (wait_event_timeout(doe_mb->wq,
-			       test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
-			       timeout))
-		return -EIO;
-	return 0;
-}
-
 static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
 {
 	struct pci_dev *pdev = doe_mb->pdev;
@@ -82,12 +72,9 @@ static int pci_doe_abort(struct pci_doe_mb *doe_mb)
 	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
 
 	do {
-		int rc;
 		u32 val;
 
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc)
-			return rc;
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
 
 		/* Abort success! */
@@ -278,11 +265,7 @@ static void doe_statemachine_work(struct work_struct *work)
 			signal_task_abort(task, -EIO);
 			return;
 		}
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc) {
-			signal_task_abort(task, rc);
-			return;
-		}
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		goto retry_resp;
 	}
 
@@ -383,21 +366,6 @@ static void pci_doe_destroy_workqueue(void *mb)
 	destroy_workqueue(doe_mb->work_queue);
 }
 
-static void pci_doe_flush_mb(void *mb)
-{
-	struct pci_doe_mb *doe_mb = mb;
-
-	/* Stop all pending work items from starting */
-	set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
-
-	/* Cancel an in progress work item, if necessary */
-	set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
-	wake_up(&doe_mb->wq);
-
-	/* Flush all work items */
-	flush_workqueue(doe_mb->work_queue);
-}
-
 /**
  * pcim_doe_create_mb() - Create a DOE mailbox object
  *
@@ -450,14 +418,6 @@ struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
 		return ERR_PTR(rc);
 	}
 
-	/*
-	 * The state machine and the mailbox should be in sync now;
-	 * Set up mailbox flush prior to using the mailbox to query protocols.
-	 */
-	rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
-	if (rc)
-		return ERR_PTR(rc);
-
 	rc = pci_doe_cache_protocols(doe_mb);
 	if (rc) {
 		pci_err(pdev, "[%x] failed to cache protocols : %d\n",
#4 lore raw

Re: [PATCH V2 1/2] PCI/DOE: Remove the pci_doe_flush_mb() call

Quoted 44 lines · On Wed, 23 Nov 2022 09:35:37 -0800 Ira Weiny <[email protected]> wrote:
Quoted 36 lines · Jonathan Cameron
Quoted 14 lines · On Tue, 22 Nov 2022 07:53:23 -0800 [email protected] wrote:

From: Ira Weiny <[email protected]>

Each struct doe_mb is managed as part of the PCI device. They can't go away as long as the PCI device exists. pci_doe_flush_mb() was set up to flush the workqueue and prevent any further submissions to the mailboxes when the PCI device goes away. Unfortunately, this was fundamentally flawed. There was no guarantee that a struct doe_mb remained after pci_doe_flush_mb() returned. Therefore, the doe_mb state could be invalid when those threads waiting on the workqueue were flushed.

Fortunately the current code is safe because all callers make a synchronous call to pci_doe_submit_task() and maintain a reference on the PCI device.

Quoted 6 lines

For these reasons, pci_doe_flush_mb() will never be called while tasks are being processed and there is no use for it.

Remove the dead code around pci_doe_flush_mb().

Looks fine I think, though one question inline.

Quoted 10 lines
---
 drivers/pci/doe.c | 48 ++++-------------------------------------------
 1 file changed, 4 insertions(+), 44 deletions(-)
drivers/pci/doe.c +0 -0
index e402f05068a5..260313e9052e 100644
@@ -24,10 +24,9 @@
 
Quoted 6 lines

drivers/pci/doe.c line 27

Quoted 4 lines
drivers/pci/doe.c +1 -1
 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
 #define PCI_DOE_TIMEOUT HZ
-#define PCI_DOE_POLL_INTERVAL	(PCI_DOE_TIMEOUT / 128)
+#define PCI_DOE_POLL_INTERVAL	8  

Why this change?

Quoted 6 lines

msleep_interruptible() takes a millisecond value and wait_event_timeout() takes jiffies. 1/128 of a second is ~8ms.

While for most configs (HZ == 1000) the value does not change. I don't believe this would be true for all configs. Thus a more explicit define.

Makes sense. Maybe add a postfix as well to make it clear it's not in same units as the PCI_DOE_TIMEOUT?
PCI_DOE_POLL_INTERVAL_MSECS

Quoted 94 lines

I'll add a comment.

Ira

Quoted 89 lines
Quoted 88 lines
drivers/pci/doe.c +3 -43
-#define PCI_DOE_FLAG_CANCEL	0
-#define PCI_DOE_FLAG_DEAD	1
+#define PCI_DOE_FLAG_DEAD	0
 
 /**
  * struct pci_doe_mb - State for a single DOE mailbox
@@ -53,15 +52,6 @@ struct pci_doe_mb {
 	unsigned long flags;
 };
 
-static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
-{
-	if (wait_event_timeout(doe_mb->wq,
-			       test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
-			       timeout))
-		return -EIO;
-	return 0;
-}
-
 static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
 {
 	struct pci_dev *pdev = doe_mb->pdev;
@@ -82,12 +72,9 @@ static int pci_doe_abort(struct pci_doe_mb *doe_mb)
 	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
 
 	do {
-		int rc;
 		u32 val;
 
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc)
-			return rc;
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
 
 		/* Abort success! */
@@ -278,11 +265,7 @@ static void doe_statemachine_work(struct work_struct *work)
 			signal_task_abort(task, -EIO);
 			return;
 		}
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
-		if (rc) {
-			signal_task_abort(task, rc);
-			return;
-		}
+		msleep_interruptible(PCI_DOE_POLL_INTERVAL);
 		goto retry_resp;
 	}
 
@@ -383,21 +366,6 @@ static void pci_doe_destroy_workqueue(void *mb)
 	destroy_workqueue(doe_mb->work_queue);
 }
 
-static void pci_doe_flush_mb(void *mb)
-{
-	struct pci_doe_mb *doe_mb = mb;
-
-	/* Stop all pending work items from starting */
-	set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
-
-	/* Cancel an in progress work item, if necessary */
-	set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
-	wake_up(&doe_mb->wq);
-
-	/* Flush all work items */
-	flush_workqueue(doe_mb->work_queue);
-}
-
 /**
  * pcim_doe_create_mb() - Create a DOE mailbox object
  *
@@ -450,14 +418,6 @@ struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
 		return ERR_PTR(rc);
 	}
 
-	/*
-	 * The state machine and the mailbox should be in sync now;
-	 * Set up mailbox flush prior to using the mailbox to query protocols.
-	 */
-	rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
-	if (rc)
-		return ERR_PTR(rc);
-
 	rc = pci_doe_cache_protocols(doe_mb);
 	if (rc) {
 		pci_err(pdev, "[%x] failed to cache protocols : %d\n",  
#5 lore raw

Re: [PATCH V2 1/2] PCI/DOE: Remove the pci_doe_flush_mb() call

In reply to [email protected]
Quoted 8 lines · On Tue, Nov 22, 2022 at 07:53:23AM -0800, [email protected] wrote:

Each struct doe_mb is managed as part of the PCI device. They can't go away as long as the PCI device exists. pci_doe_flush_mb() was set up to flush the workqueue and prevent any further submissions to the mailboxes when the PCI device goes away. Unfortunately, this was fundamentally flawed. There was no guarantee that a struct doe_mb remained after pci_doe_flush_mb() returned. Therefore, the doe_mb state could be invalid when those threads waiting on the workqueue were flushed.

Quoted 6 lines

Fortunately the current code is safe because all callers make a synchronous call to pci_doe_submit_task() and maintain a reference on the PCI device.

For these reasons, pci_doe_flush_mb() will never be called while tasks are being processed and there is no use for it.

Going forward my plan is to allocate all existing DOE mailboxes of a device upon enumeration. That will allow concurrent use of a mailbox by multiple drivers.

When a pci_dev goes away, say, because it's been hot-removed, we need a way to abort all ongoing DOE exchanges.

pci_doe_flush_mb() seems to do just that so I'm not sure why it's being removed?

Thanks,

Lukas

author
#6 lore raw

Re: [PATCH V2 1/2] PCI/DOE: Remove the pci_doe_flush_mb() call

Quoted 20 lines · Lukas Wunner
Quoted 8 lines · On Tue, Nov 22, 2022 at 07:53:23AM -0800, [email protected] wrote:

Each struct doe_mb is managed as part of the PCI device. They can't go away as long as the PCI device exists. pci_doe_flush_mb() was set up to flush the workqueue and prevent any further submissions to the mailboxes when the PCI device goes away. Unfortunately, this was fundamentally flawed. There was no guarantee that a struct doe_mb remained after pci_doe_flush_mb() returned. Therefore, the doe_mb state could be invalid when those threads waiting on the workqueue were flushed.

Quoted 6 lines

Fortunately the current code is safe because all callers make a synchronous call to pci_doe_submit_task() and maintain a reference on the PCI device.

For these reasons, pci_doe_flush_mb() will never be called while tasks are being processed and there is no use for it.

Going forward my plan is to allocate all existing DOE mailboxes of a device upon enumeration. That will allow concurrent use of a mailbox by multiple drivers.

Quoted 5 lines

When a pci_dev goes away, say, because it's been hot-removed, we need a way to abort all ongoing DOE exchanges.

pci_doe_flush_mb() seems to do just that so I'm not sure why it's being removed?

I guess I was not explicit enough in the commit message. 1) it is not used today. More importantly 2) it does not work.

Because we are not using it currently I thought it best to remove it rather than try and fix it. Once we have a use then we can figure out how to make sure the doe_mb[*] is valid until all tasks are flushed.

Ira

[*] Probably with a reference on the struct as was discussed before.

	-- https://lore.kernel.org/all/[email protected]/
Quoted 4 lines

Thanks,

Lukas

#7 lore raw

PATCH v2 2/2 PCI/DOE: Remove asynchronous task support

In reply to [email protected]

From: Ira Weiny <[email protected]>

Gregory Price and Jonathan Cameron reported a bug within pci_doe_submit_task().[1] The issue was that work item initialization needs to be done with either INIT_WORK_ONSTACK() or INIT_WORK() depending on how the work item is allocated.

Initially, it was anticipated that DOE tasks were going to need to be submitted asynchronously and the code was designed thusly. Many alternatives were discussed to fix the work initialization issue.[2]

However, all current users submit tasks synchronously and this has therefore become an unneeded maintenance burden. Remove the extra maintenance burden by replacing asynchronous task submission with a synchronous wait function.[3]

[1] https://lore.kernel.org/linux-cxl/[email protected]/T/#m88a7f50dcce52f30c8bf5c3dcc06fa9843b54a2d
[2] https://lore.kernel.org/linux-cxl/Y3kSDQDur+IUDs2O@iweiny-mobl/T/#m0f057773d9c75432fcfcc54a2604483fe82abe92
[3] https://lore.kernel.org/linux-cxl/Y3kSDQDur+IUDs2O@iweiny-mobl/T/#m32d3f9b208ef7486bc148d94a326b26b2d3e69ff

---
Changes from V1:
Dan Williams:
Use mutex_lock_interruptible()
Adjust comment to lock the doe_mb structure not the code
Adjust the locking based on the new series which eliminated PCI_DOE_FLAG_CANCEL.

Thanks to Dan for the bulk of the patch.
Thanks to Ming for pointing out the need for a lock to prevent more than 1 task from being processed at a time.

---
 drivers/cxl/core/pci.c  | 16 ++-------
 drivers/pci/doe.c       | 75 +++++++++++++----------------------------
 include/linux/pci-doe.h | 10 +-----
 3 files changed, 28 insertions(+), 73 deletions(-)
drivers/cxl/core/pci.c +3 -13
index 9240df53ed87..58977e0712b6 100644
@@ -490,21 +490,14 @@ static struct pci_doe_mb *find_cdat_doe(struct device *uport)
 		    CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) |		\
 	 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle)))
 
-static void cxl_doe_task_complete(struct pci_doe_task *task)
-{
-	complete(task->private);
-}
-
 struct cdat_doe_task {
 	u32 request_pl;
 	u32 response_pl[32];
-	struct completion c;
 	struct pci_doe_task task;
 };
 
 #define DECLARE_CDAT_DOE_TASK(req, cdt)                       \
 struct cdat_doe_task cdt = {                                  \
-	.c = COMPLETION_INITIALIZER_ONSTACK(cdt.c),           \
 	.request_pl = req,				      \
 	.task = {                                             \
 		.prot.vid = PCI_DVSEC_VENDOR_ID_CXL,        \
@@ -513,8 +506,6 @@ struct cdat_doe_task cdt = {                                  \
 		.request_pl_sz = sizeof(cdt.request_pl),      \
 		.response_pl = cdt.response_pl,               \
 		.response_pl_sz = sizeof(cdt.response_pl),    \
-		.complete = cxl_doe_task_complete,            \
-		.private = &cdt.c,                            \
 	}                                                     \
 }
 
@@ -525,12 +516,12 @@ static int cxl_cdat_get_length(struct device *dev,
 	DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(0), t);
 	int rc;
 
-	rc = pci_doe_submit_task(cdat_doe, &t.task);
+	rc = pci_doe_submit_task_wait(cdat_doe, &t.task);
 	if (rc < 0) {
 		dev_err(dev, "DOE submit failed: %d", rc);
 		return rc;
 	}
-	wait_for_completion(&t.c);
+
 	if (t.task.rv < sizeof(u32))
 		return -EIO;
 
@@ -554,12 +545,11 @@ static int cxl_cdat_read_table(struct device *dev,
 		u32 *entry;
 		int rc;
 
-		rc = pci_doe_submit_task(cdat_doe, &t.task);
+		rc = pci_doe_submit_task_wait(cdat_doe, &t.task);
 		if (rc < 0) {
 			dev_err(dev, "DOE submit failed: %d", rc);
 			return rc;
 		}
-		wait_for_completion(&t.c);
 		/* 1 DW header + 1 DW data min */
 		if (t.task.rv < (2 * sizeof(u32)))
 			return -EIO;
drivers/pci/doe.c +24 -51

167 lines not on this page · raw message

include/linux/pci-doe.h +1 -9
index ed9b4df792b8..c94122a66221 100644
@@ -30,8 +30,6 @@ struct pci_doe_mb;
  * @response_pl_sz: Size of the response payload (bytes)
  * @rv: Return value.  Length of received response or error (bytes)
  * @complete: Called when task is complete
- * @private: Private data for the consumer
- * @work: Used internally by the mailbox
  * @doe_mb: Used internally by the mailbox
  *
  * The payload sizes and rv are specified in bytes with the following
@@ -50,11 +48,6 @@ struct pci_doe_task {
 	u32 *response_pl;
 	size_t response_pl_sz;
 	int rv;
-	void (*complete)(struct pci_doe_task *task);
-	void *private;
-
-	/* No need for the user to initialize these fields */
-	struct work_struct work;
 	struct pci_doe_mb *doe_mb;
 };
 
@@ -72,6 +65,5 @@ struct pci_doe_task {
 
 struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset);
 bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type);
-int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task);
-
+int pci_doe_submit_task_wait(struct pci_doe_mb *doe_mb, struct pci_doe_task *task);
 #endif
Signature · 1 line
2.37.2
#8 lore raw
Quoted 20 lines · On Tue, 22 Nov 2022 07:53:24 -0800 [email protected] wrote:

From: Ira Weiny <[email protected]>

Gregory Price and Jonathan Cameron reported a bug within pci_doe_submit_task().[1] The issue was that work item initialization needs to be done with either INIT_WORK_ONSTACK() or INIT_WORK() depending on how the work item is allocated.

Initially, it was anticipated that DOE tasks were going to need to be submitted asynchronously and the code was designed thusly. Many alternatives were discussed to fix the work initialization issue.[2]

However, all current users submit tasks synchronously and this has therefore become an unneeded maintenance burden. Remove the extra maintenance burden by replacing asynchronous task submission with a synchronous wait function.[3]

[1] https://lore.kernel.org/linux-cxl/[email protected]/T/#m88a7f50dcce52f30c8bf5c3dcc06fa9843b54a2d
[2] https://lore.kernel.org/linux-cxl/Y3kSDQDur+IUDs2O@iweiny-mobl/T/#m0f057773d9c75432fcfcc54a2604483fe82abe92
[3] https://lore.kernel.org/linux-cxl/Y3kSDQDur+IUDs2O@iweiny-mobl/T/#m32d3f9b208ef7486bc148d94a326b26b2d3e69ff

Quoted 6 lines

Hi Ira,

A few things inline.

Jonathan

Quoted 31 lines
drivers/pci/doe.c +2 -3
index 260313e9052e..41c7bf5794a5 100644
@@ -18,7 +18,6 @@
 #include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/pci-doe.h>
-#include <linux/workqueue.h>
 
 #define PCI_DOE_PROTOCOL_DISCOVERY 0
 
@@ -39,7 +38,7 @@
  * @cap_offset: Capability offset
  * @prots: Array of protocols supported (encoded as long values)
  * @wq: Wait queue for work item
- * @work_queue: Queue of pci_doe_work items
+ * @lock: Lock state of doe_mb during task processing
  * @flags: Bit array of PCI_DOE_FLAG_* flags
  */
 struct pci_doe_mb {
@@ -48,7 +47,7 @@ struct pci_doe_mb {
 	struct xarray prots;
 
 	wait_queue_head_t wq;
-	struct workqueue_struct *work_queue;
+	struct mutex lock;
 	unsigned long flags;
 };
 

drivers/pci/doe.c static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *tas · line 197

Quoted 1 line
drivers/pci/doe.c +0 -0
@@ -198,7 +197,6 @@ static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *tas
 static void signal_task_complete(struct pci_doe_task *task, int rv)

Given this doesn't signal anything any more, perhaps rename the function, or just push the task->rv = ... inline.?

Quoted 4 lines
drivers/pci/doe.c +0 -1
 {
 	task->rv = rv;
-	task->complete(task);
 }

...

include/linux/pci-doe.h struct pci_doe_mb; · line 32

Quoted 8 lines
include/linux/pci-doe.h +0 -0
index ed9b4df792b8..c94122a66221 100644
@@ -30,8 +30,6 @@ struct pci_doe_mb;
  * @response_pl_sz: Size of the response payload (bytes)
  * @rv: Return value.  Length of received response or error (bytes)
  * @complete: Called when task is complete

complete is gone as well.

Quoted 16 lines
include/linux/pci-doe.h +0 -7
- * @private: Private data for the consumer
- * @work: Used internally by the mailbox
  * @doe_mb: Used internally by the mailbox
  *
  * The payload sizes and rv are specified in bytes with the following
@@ -50,11 +48,6 @@ struct pci_doe_task {
 	u32 *response_pl;
 	size_t response_pl_sz;
 	int rv;
-	void (*complete)(struct pci_doe_task *task);
-	void *private;
-
-	/* No need for the user to initialize these fields */
-	struct work_struct work;
 	struct pci_doe_mb *doe_mb;
 };

...