Skip to contentSkip to conversation

qed: Fix and enhance slowpath workqueue mechanism.

PATCH In review Yuval Basson 2 participants 6 messages Mar 24, 2020 – Mar 25, 2020

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover qed: Fix and enhance slowpath workqueue mechanism.

The next patch series enhances slowpath workqueue mechanism and fixes a race between scheduled tasks and destroy_workqueue.

Yuval Basson (3):
qed: Replace wq_active Boolean with an atomic QED_SLOWPATH_ACTIVE flag
qed: Add a flag for rescheduling the slowpath task
qed: Fix race condition between scheduling and destroying the slowpath
workqueue

 drivers/net/ethernet/qlogic/qed/qed.h      |  3 ++-
 drivers/net/ethernet/qlogic/qed/qed_main.c | 37 +++++++++++++-----------------
 2 files changed, 18 insertions(+), 22 deletions(-)
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH 1/3 qed: Replace wq_active Boolean with an atomic QED_SLOWPATH_ACTIVE flag

The atomic opertaion might prevent a potential race condition.

---
 drivers/net/ethernet/qlogic/qed/qed.h      | 2 +-
 drivers/net/ethernet/qlogic/qed/qed_main.c | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)
drivers/net/ethernet/qlogic/qed/qed.h +1 -1
index fa41bf0..ca866c2 100644
@@ -565,6 +565,7 @@ struct qed_simd_fp_handler {
 enum qed_slowpath_wq_flag {
 	QED_SLOWPATH_MFW_TLV_REQ,
 	QED_SLOWPATH_PERIODIC_DB_REC,
+	QED_SLOWPATH_ACTIVE,
 };
 
 struct qed_hwfn {
@@ -700,7 +701,6 @@ struct qed_hwfn {
 	unsigned long iov_task_flags;
 #endif
 	struct z_stream_s *stream;
-	bool slowpath_wq_active;
 	struct workqueue_struct *slowpath_wq;
 	struct delayed_work slowpath_task;
 	unsigned long slowpath_task_flags;
drivers/net/ethernet/qlogic/qed/qed_main.c +5 -4
index 2c189c6..016d658 100644
@@ -1095,7 +1095,7 @@ static int qed_slowpath_delayed_work(struct qed_hwfn *hwfn,
 				     enum qed_slowpath_wq_flag wq_flag,
 				     unsigned long delay)
 {
-	if (!hwfn->slowpath_wq_active)
+	if (!test_bit(QED_SLOWPATH_ACTIVE, &hwfn->slowpath_task_flags))
 		return -EINVAL;
 
 	/* Memory barrier for setting atomic bit */
@@ -1133,7 +1133,8 @@ static void qed_slowpath_wq_stop(struct qed_dev *cdev)
 			continue;
 
 		/* Stop queuing new delayed works */
-		cdev->hwfns[i].slowpath_wq_active = false;
+		clear_bit(QED_SLOWPATH_ACTIVE,
+			  &cdev->hwfns[i].slowpath_task_flags);
 
 		/* Wait until the last periodic doorbell recovery is executed */
 		while (test_bit(QED_SLOWPATH_PERIODIC_DB_REC,
@@ -1153,7 +1154,7 @@ static void qed_slowpath_task(struct work_struct *work)
 	struct qed_ptt *ptt = qed_ptt_acquire(hwfn);
 
 	if (!ptt) {
-		if (hwfn->slowpath_wq_active)
+		if (test_bit(QED_SLOWPATH_ACTIVE, &hwfn->slowpath_task_flags))
 			queue_delayed_work(hwfn->slowpath_wq,
 					   &hwfn->slowpath_task, 0);
 
@@ -1199,7 +1200,7 @@ static int qed_slowpath_wq_start(struct qed_dev *cdev)
 		}
 
 		INIT_DELAYED_WORK(&hwfn->slowpath_task, qed_slowpath_task);
-		hwfn->slowpath_wq_active = true;
+		set_bit(QED_SLOWPATH_ACTIVE, &hwfn->slowpath_task_flags);
 	}
 
 	return 0;
Signature · 1 line
1.8.3.1
maintainer
#2 lore raw

Re: [PATCH net-next 1/3] qed: Replace wq_active Boolean with an atomic QED_SLOWPATH_ACTIVE flag

From: Yuval Basson <[email protected]>
Date: Tue, 24 Mar 2020 16:13:46 +0200

Quoted 4 lines

The atomic opertaion might prevent a potential race condition.

There is no basis in fact behind this change.

Either explain clearly and precisely what race is fixed by this change or remove this change.

Thank you.

#3 lore raw

RE: [EXT] Re: [PATCH net-next 1/3] qed: Replace wq_active Boolean with an atomic QED_SLOWPATH_ACTIVE flag

Quoted 28 lines

-----Original Message-----
From: David Miller <[email protected]>
Sent: Wednesday, March 25, 2020 1:36 AM
To: Yuval Basson <[email protected]>
Cc: [email protected]; Denis Bolotin <[email protected]>
Subject: [EXT] Re: [PATCH net-next 1/3] qed: Replace wq_active Boolean with an atomic QED_SLOWPATH_ACTIVE flag

External Email

----------------------------------------------------------------------
From: Yuval Basson <[email protected]>
Date: Tue, 24 Mar 2020 16:13:46 +0200

Quoted 4 lines

The atomic opertaion might prevent a potential race condition.

There is no basis in fact behind this change.

Quoted 2 lines

thanks for the comment, indeed it seems the only patch required to solve the race is the third one Sending a v2 with just the third patch.

Either explain clearly and precisely what race is fixed by this change or remove this change.

Thank you.

#4 lore raw

PATCH 2/3 qed: Add a flag for rescheduling the slowpath task

In reply to Yuval Basson

Rechedule delayed work in case ptt_acquire failed. Since it is the same task don't reset task's flags.

---
 drivers/net/ethernet/qlogic/qed/qed.h      |  1 +
 drivers/net/ethernet/qlogic/qed/qed_main.c | 15 +++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)
drivers/net/ethernet/qlogic/qed/qed.h +1 -0
index ca866c2..e3b238e 100644
@@ -566,6 +566,7 @@ enum qed_slowpath_wq_flag {
 	QED_SLOWPATH_MFW_TLV_REQ,
 	QED_SLOWPATH_PERIODIC_DB_REC,
 	QED_SLOWPATH_ACTIVE,
+	QED_SLOWPATH_RESCHEDULE,
 };
 
 struct qed_hwfn {
drivers/net/ethernet/qlogic/qed/qed_main.c +9 -6
index 016d658..fb13863 100644
@@ -1098,10 +1098,13 @@ static int qed_slowpath_delayed_work(struct qed_hwfn *hwfn,
 	if (!test_bit(QED_SLOWPATH_ACTIVE, &hwfn->slowpath_task_flags))
 		return -EINVAL;
 
-	/* Memory barrier for setting atomic bit */
-	smp_mb__before_atomic();
-	set_bit(wq_flag, &hwfn->slowpath_task_flags);
-	smp_mb__after_atomic();
+	if (wq_flag != QED_SLOWPATH_RESCHEDULE) {
+		/* Memory barrier for setting atomic bit */
+		smp_mb__before_atomic();
+		set_bit(wq_flag, &hwfn->slowpath_task_flags);
+		smp_mb__after_atomic();
+	}
+
 	queue_delayed_work(hwfn->slowpath_wq, &hwfn->slowpath_task, delay);
 
 	return 0;
@@ -1155,8 +1158,8 @@ static void qed_slowpath_task(struct work_struct *work)
 
 	if (!ptt) {
 		if (test_bit(QED_SLOWPATH_ACTIVE, &hwfn->slowpath_task_flags))
-			queue_delayed_work(hwfn->slowpath_wq,
-					   &hwfn->slowpath_task, 0);
+			qed_slowpath_delayed_work(hwfn,
+						  QED_SLOWPATH_RESCHEDULE, 0);
 
 		return;
 	}
Signature · 1 line
1.8.3.1
#5 lore raw

PATCH 3/3 qed: Fix race condition between scheduling and destroying the slowpath workqueue

In reply to Yuval Basson

Calling queue_delayed_work concurrently with destroy_workqueue might race to an unexpected outcome - scheduled task after wq is destroyed or other resources (like ptt_pool) are freed (yields NULL pointer dereference). cancel_delayed_work prevents the race by cancelling the timer triggered for scheduling a new task.

---
 drivers/net/ethernet/qlogic/qed/qed_main.c | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)
drivers/net/ethernet/qlogic/qed/qed_main.c +2 -11
index fb13863..ade927d 100644
@@ -1087,9 +1087,6 @@ static void qed_update_pf_params(struct qed_dev *cdev,
 #define QED_PERIODIC_DB_REC_INTERVAL_MS		100
 #define QED_PERIODIC_DB_REC_INTERVAL \
 	msecs_to_jiffies(QED_PERIODIC_DB_REC_INTERVAL_MS)
-#define QED_PERIODIC_DB_REC_WAIT_COUNT		10
-#define QED_PERIODIC_DB_REC_WAIT_INTERVAL \
-	(QED_PERIODIC_DB_REC_INTERVAL_MS / QED_PERIODIC_DB_REC_WAIT_COUNT)
 
 static int qed_slowpath_delayed_work(struct qed_hwfn *hwfn,
 				     enum qed_slowpath_wq_flag wq_flag,
@@ -1126,7 +1123,7 @@ void qed_periodic_db_rec_start(struct qed_hwfn *p_hwfn)
 
 static void qed_slowpath_wq_stop(struct qed_dev *cdev)
 {
-	int i, sleep_count = QED_PERIODIC_DB_REC_WAIT_COUNT;
+	int i;
 
 	if (IS_VF(cdev))
 		return;
@@ -1139,13 +1136,7 @@ static void qed_slowpath_wq_stop(struct qed_dev *cdev)
 		clear_bit(QED_SLOWPATH_ACTIVE,
 			  &cdev->hwfns[i].slowpath_task_flags);
 
-		/* Wait until the last periodic doorbell recovery is executed */
-		while (test_bit(QED_SLOWPATH_PERIODIC_DB_REC,
-				&cdev->hwfns[i].slowpath_task_flags) &&
-		       sleep_count--)
-			msleep(QED_PERIODIC_DB_REC_WAIT_INTERVAL);
-
-		flush_workqueue(cdev->hwfns[i].slowpath_wq);
+		cancel_delayed_work(&cdev->hwfns[i].slowpath_task);
 		destroy_workqueue(cdev->hwfns[i].slowpath_wq);
 	}
 }
Signature · 1 line
1.8.3.1