dmaengine: idxd: Fix up re-enabling device workqueues
Subsystems (2)
cover dmaengine: idxd: Fix up re-enabling device workqueues
Currently if a software reset is attempted on an idxd device the workqueues will not be re-enabled, because it will see incorrectly see that wq->state is already set to IDXD_WQ_ENABLED. So set the workqueue state to disabled in idxd_wq_disable_cleanup(), and use a bitmap to track which workqueues have been enabled so they can be re-enabled during device re-initialization.
Jerry Snitselaar (2):
dmaengine: idxd: Set wq state to disabled in idxd_wq_disable_cleanup()
dmaengine: idxd: track enabled workqueues in bitmap
drivers/dma/idxd/device.c | 4 +++- drivers/dma/idxd/idxd.h | 2 ++ drivers/dma/idxd/init.c | 6 ++++++ drivers/dma/idxd/irq.c | 4 ++-- drivers/dma/idxd/sysfs.c | 1 + 5 files changed, 14 insertions(+), 3 deletions(-)
Signature · 1 line
2.37.2
PATCH 1/2 dmaengine: idxd: Set wq state to disabled in idxd_wq_disable_cleanup()
If we are calling idxd_wq_disable_cleanup(), the workqueue should be in a disabled state. So set the workqueue state to IDXD_WQ_DISABLED so that the state reflects that. Currently if there is a device failure, and a software reset is attempted the workqueues will not be re-enabled due to idxd_wq_enable() seeing that state as already being IDXD_WQ_ENABLED.
--- drivers/dma/idxd/device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/dma/idxd/device.c +1 -1
@@ -258,7 +258,6 @@ void idxd_wq_reset(struct idxd_wq *wq) operand =BIT (wq->id %16 ) | ((wq->id /16 ) <<16 );idxd_cmd_exec (idxd,IDXD_CMD_RESET_WQ , operand,NULL );idxd_wq_disable_cleanup (wq); - wq->state =IDXD_WQ_DISABLED ; }int idxd_wq_map_portal (struct idxd_wq *wq)
@@ -378,6 +377,7 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq)struct idxd_device *idxd = wq->idxd;lockdep_assert_held (&wq->wq_lock); + wq->state =IDXD_WQ_DISABLED ;memset (wq->wqcfg,0 , idxd->wqcfg_size); wq->type =IDXD_WQT_NONE ; wq->threshold =0 ;
Signature · 1 line
2.37.2
PATCH 2/2 dmaengine: idxd: track enabled workqueues in bitmap
In reply to Jerry SnitselaarNow that idxd_wq_disable_cleanup() sets the workqueue state to IDXD_WQ_DISABLED, use a bitmap to track which workqueues have been enabled. This will then be used to determine which workqueues should be re-enabled when attempting a software reset to recover from a device halt state.
--- drivers/dma/idxd/device.c | 2 ++ drivers/dma/idxd/idxd.h | 2 ++ drivers/dma/idxd/init.c | 6 ++++++ drivers/dma/idxd/irq.c | 4 ++-- drivers/dma/idxd/sysfs.c | 1 + 5 files changed, 13 insertions(+), 2 deletions(-)
drivers/dma/idxd/device.c +2 -0
@@ -196,6 +196,7 @@ int idxd_wq_enable(struct idxd_wq *wq) } wq->state =IDXD_WQ_ENABLED ; +set_bit (wq->id, idxd->wq_enable_map);dev_dbg (dev,"WQ %d enabled\n" , wq->id);return 0 ; }
@@ -223,6 +224,7 @@ int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)if (reset_config)idxd_wq_disable_cleanup (wq); +clear_bit (wq->id, idxd->wq_enable_map); wq->state =IDXD_WQ_DISABLED ;dev_dbg (dev,"WQ %d disabled\n" , wq->id);return 0 ;
drivers/dma/idxd/idxd.h +2 -0
@@ -11,6 +11,7 @@#include <linux/idr.h> #include <linux/pci.h> #include <linux/ioasid.h> +#include <linux/bitmap.h> #include <linux/perf_event.h> #include <uapi/linux/idxd.h> #include "registers.h"
@@ -299,6 +300,7 @@ struct idxd_device {int rdbuf_limit;int nr_rdbufs;/* non-reserved read buffers */ unsigned int wqcfg_size; +unsigned long *wq_enable_map;union sw_err_reg sw_err;wait_queue_head_t cmd_waitq;
drivers/dma/idxd/init.c +6 -0
@@ -151,6 +151,12 @@ static int idxd_setup_wqs(struct idxd_device *idxd)if (!idxd->wqs)return -ENOMEM ; + idxd->wq_enable_map =bitmap_zalloc_node (idxd->max_wqs,GFP_KERNEL ,dev_to_node (dev)); +if (!idxd->wq_enable_map) { +kfree (idxd->wqs); +return -ENOMEM ; + } +for (i =0 ; i < idxd->max_wqs; i++) { wq =kzalloc_node (sizeof (*wq),GFP_KERNEL ,dev_to_node (dev));if (!wq) {
drivers/dma/idxd/irq.c +2 -2
@@ -49,9 +49,9 @@ static void idxd_device_reinit(struct work_struct *work) goto out;for (i =0 ; i < idxd->max_wqs; i++) { -struct idxd_wq *wq = idxd->wqs[i]; +if (test_bit (i, idxd->wq_enable_map)) { +struct idxd_wq *wq = idxd->wqs[i]; -if (wq->state ==IDXD_WQ_ENABLED ) { rc =idxd_wq_enable (wq);if (rc <0 ) { dev_warn(dev, "Unable to re-enable wq %s\n",
drivers/dma/idxd/sysfs.c +1 -0
@@ -1405,6 +1405,7 @@ static void idxd_conf_device_release(struct device *dev)struct idxd_device *idxd =confdev_to_idxd (dev);kfree (idxd->groups); +bitmap_free (idxd->wq_enable_map);kfree (idxd->wqs);kfree (idxd->engines);ida_free (&idxd_ida, idxd->id);
Signature · 1 line
2.37.2
Re: [PATCH 2/2] dmaengine: idxd: track enabled workqueues in bitmap
Quoted 81 lines · Jerry Snitselaar
Now that
idxd_wq_disable_cleanup()sets the workqueue state toIDXD_WQ_DISABLED, use a bitmap to track which workqueues have been enabled. This will then be used to determine which workqueues should be re-enabled when attempting a software reset to recover from a device halt state.Cc Fenghua Yu <[email protected]>Cc Dave Jiang <[email protected]>Cc Vinod Koul <[email protected]>Signed-off-by Jerry Snitselaar <[email protected]>--- drivers/dma/idxd/device.c | 2 ++ drivers/dma/idxd/idxd.h | 2 ++ drivers/dma/idxd/init.c | 6 ++++++ drivers/dma/idxd/irq.c | 4 ++-- drivers/dma/idxd/sysfs.c | 1 + 5 files changed, 13 insertions(+), 2 deletions(-)drivers/dma/idxd/device.c +2
-0@@ -196,6 +196,7 @@ int idxd_wq_enable(struct idxd_wq *wq) } wq->state =IDXD_WQ_ENABLED ; +set_bit (wq->id, idxd->wq_enable_map);dev_dbg (dev,"WQ %d enabled\n" , wq->id);return 0 ; }@@ -223,6 +224,7 @@ int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)if (reset_config)idxd_wq_disable_cleanup (wq); +clear_bit (wq->id, idxd->wq_enable_map); wq->state =IDXD_WQ_DISABLED ;dev_dbg (dev,"WQ %d disabled\n" , wq->id);return 0 ;drivers/dma/idxd/idxd.h +2
-0@@ -11,6 +11,7 @@#include <linux/idr.h> #include <linux/pci.h> #include <linux/ioasid.h> +#include <linux/bitmap.h> #include <linux/perf_event.h> #include <uapi/linux/idxd.h> #include "registers.h" @@ -299,6 +300,7 @@ struct idxd_device {int rdbuf_limit;int nr_rdbufs;/* non-reserved read buffers */ unsigned int wqcfg_size; +unsigned long *wq_enable_map;union sw_err_reg sw_err;wait_queue_head_t cmd_waitq;drivers/dma/idxd/init.c +6
-0@@ -151,6 +151,12 @@ static int idxd_setup_wqs(struct idxd_device *idxd)if (!idxd->wqs)return -ENOMEM ; + idxd->wq_enable_map =bitmap_zalloc_node (idxd->max_wqs,GFP_KERNEL ,dev_to_node (dev)); +if (!idxd->wq_enable_map) { +kfree (idxd->wqs); +return -ENOMEM ; + } +for (i =0 ; i < idxd->max_wqs; i++) { wq =kzalloc_node (sizeof (*wq),GFP_KERNEL ,dev_to_node (dev));if (!wq) {drivers/dma/idxd/irq.c +0
-0@@ -49,9 +49,9 @@ static void idxd_device_reinit(struct work_struct *work) goto out;
drivers/dma/idxd/irq.c static void idxd_device_reinit(struct work_struct *work) · line 57
Quoted 10 lines
drivers/dma/idxd/irq.c +2
-2for (i =0 ; i < idxd->max_wqs; i++) { -struct idxd_wq *wq = idxd->wqs[i]; +if (test_bit (i, idxd->wq_enable_map)) { +struct idxd_wq *wq = idxd->wqs[i]; -if (wq->state ==IDXD_WQ_ENABLED ) { rc =idxd_wq_enable (wq);if (rc <0 ) { dev_warn(dev, "Unable to re-enable wq %s\n",
Hi Dave and Fenghua,
Thinking about this last night, this should probably clear the bit here in the case where an error is returned from idxd_wq_enable here, yes? I can send a v2.
Quoted 15 lines
drivers/dma/idxd/sysfs.c +1
-0@@ -1405,6 +1405,7 @@ static void idxd_conf_device_release(struct device *dev)struct idxd_device *idxd =confdev_to_idxd (dev);kfree (idxd->groups); +bitmap_free (idxd->wq_enable_map);kfree (idxd->wqs);kfree (idxd->engines);ida_free (&idxd_ida, idxd->id);Signature · 1 line
2.37.2
Dave Jiang
reviewerRe: [PATCH 2/2] dmaengine: idxd: track enabled workqueues in bitmap
Quoted 94 lines · Jerry Snitselaar
Quoted 81 lines · Jerry Snitselaar
Now that
idxd_wq_disable_cleanup()sets the workqueue state toIDXD_WQ_DISABLED, use a bitmap to track which workqueues have been enabled. This will then be used to determine which workqueues should be re-enabled when attempting a software reset to recover from a device halt state.Cc Fenghua Yu <[email protected]>Cc Dave Jiang <[email protected]>Cc Vinod Koul <[email protected]>Signed-off-by Jerry Snitselaar <[email protected]>--- drivers/dma/idxd/device.c | 2 ++ drivers/dma/idxd/idxd.h | 2 ++ drivers/dma/idxd/init.c | 6 ++++++ drivers/dma/idxd/irq.c | 4 ++-- drivers/dma/idxd/sysfs.c | 1 + 5 files changed, 13 insertions(+), 2 deletions(-)drivers/dma/idxd/device.c +2
-0@@ -196,6 +196,7 @@ int idxd_wq_enable(struct idxd_wq *wq) } wq->state =IDXD_WQ_ENABLED ; +set_bit (wq->id, idxd->wq_enable_map);dev_dbg (dev,"WQ %d enabled\n" , wq->id);return 0 ; }@@ -223,6 +224,7 @@ int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)if (reset_config)idxd_wq_disable_cleanup (wq); +clear_bit (wq->id, idxd->wq_enable_map); wq->state =IDXD_WQ_DISABLED ;dev_dbg (dev,"WQ %d disabled\n" , wq->id);return 0 ;drivers/dma/idxd/idxd.h +2
-0@@ -11,6 +11,7 @@#include <linux/idr.h> #include <linux/pci.h> #include <linux/ioasid.h> +#include <linux/bitmap.h> #include <linux/perf_event.h> #include <uapi/linux/idxd.h> #include "registers.h" @@ -299,6 +300,7 @@ struct idxd_device {int rdbuf_limit;int nr_rdbufs;/* non-reserved read buffers */ unsigned int wqcfg_size; +unsigned long *wq_enable_map;union sw_err_reg sw_err;wait_queue_head_t cmd_waitq;drivers/dma/idxd/init.c +6
-0@@ -151,6 +151,12 @@ static int idxd_setup_wqs(struct idxd_device *idxd)if (!idxd->wqs)return -ENOMEM ; + idxd->wq_enable_map =bitmap_zalloc_node (idxd->max_wqs,GFP_KERNEL ,dev_to_node (dev)); +if (!idxd->wq_enable_map) { +kfree (idxd->wqs); +return -ENOMEM ; + } +for (i =0 ; i < idxd->max_wqs; i++) { wq =kzalloc_node (sizeof (*wq),GFP_KERNEL ,dev_to_node (dev));if (!wq) {drivers/dma/idxd/irq.c +0
-0@@ -49,9 +49,9 @@ static void idxd_device_reinit(struct work_struct *work) goto out;drivers/dma/idxd/irq.c static void idxd_device_reinit(struct work_struct *work) · line 57
Quoted 10 lines
drivers/dma/idxd/irq.c +2
-2for (i =0 ; i < idxd->max_wqs; i++) { -struct idxd_wq *wq = idxd->wqs[i]; +if (test_bit (i, idxd->wq_enable_map)) { +struct idxd_wq *wq = idxd->wqs[i]; -if (wq->state ==IDXD_WQ_ENABLED ) { rc =idxd_wq_enable (wq);if (rc <0 ) { dev_warn(dev, "Unable to re-enable wq %s\n",Hi Dave and Fenghua,
Quoted 3 lines
Thinking about this last night, this should probably clear the bit here in the case where an error is returned from
idxd_wq_enablehere, yes? I can send a v2.
Ah yeah I think so. We failed to enable, so just give up. Thanks.
Quoted 16 lines
Quoted 16 lines
drivers/dma/idxd/sysfs.c +1
-0@@ -1405,6 +1405,7 @@ static void idxd_conf_device_release(struct device *dev)struct idxd_device *idxd =confdev_to_idxd (dev);kfree (idxd->groups); +bitmap_free (idxd->wq_enable_map);kfree (idxd->wqs);kfree (idxd->engines);ida_free (&idxd_ida, idxd->id);Signature · 1 line
2.37.2
Quoted 10 lines · Jerry Snitselaar
Currently if a software reset is attempted on an idxd device the workqueues will not be re-enabled, because it will see incorrectly see that
wq->stateis already set toIDXD_WQ_ENABLED. So set the workqueue state to disabled inidxd_wq_disable_cleanup(), and use a bitmap to track which workqueues have been enabled so they can be re-enabled during device re-initialization.
For the series,
Thank you Jerry!
Quoted 11 lines
Jerry Snitselaar (2):
dmaengine: idxd: Set wq state to disabled inidxd_wq_disable_cleanup()
dmaengine: idxd: track enabled workqueues in bitmapdrivers/dma/idxd/device.c | 4 +++- drivers/dma/idxd/idxd.h | 2 ++ drivers/dma/idxd/init.c | 6 ++++++ drivers/dma/idxd/irq.c | 4 ++-- drivers/dma/idxd/sysfs.c | 1 + 5 files changed, 14 insertions(+), 3 deletions(-)