crypto: qat - add heartbeat error simulator
Subsystems (2)
PATCH 1/1 crypto: qat - add heartbeat error simulator
From: Damian Muszynski <[email protected]>
Add a mechanism that allows to inject a heartbeat error for testing purposes.
A new attribute inject_error is added to debugfs for each QAT device. Upon a write on this attribute, the driver will inject an error on the device which can then be detected by the heartbeat feature. Errors are breaking the device functionality thus they require a device reset in order to be recovered.
This functionality is not compiled by default, to enable it CRYPTO_DEV_QAT_ERROR_INJECTION must be set.
--- Documentation/ABI/testing/debugfs-driver-qat | 26 +++++++ drivers/crypto/intel/qat/Kconfig | 15 ++++ drivers/crypto/intel/qat/qat_common/Makefile | 4 + .../intel/qat/qat_common/adf_common_drv.h | 3 + .../intel/qat/qat_common/adf_heartbeat.c | 6 -- .../intel/qat/qat_common/adf_heartbeat.h | 12 +++ .../qat/qat_common/adf_heartbeat_dbgfs.c | 48 ++++++++++++ .../qat/qat_common/adf_heartbeat_inject.c | 76 +++++++++++++++++++ .../intel/qat/qat_common/adf_hw_arbiter.c | 34 +++++++++ 9 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c
Documentation/ABI/testing/debugfs-driver-qat +26 -0
@@ -60,6 +60,32 @@ Description: (RO) Read returns the device health status. The driver does not monitor for Heartbeat. It is left for a user to poll the status periodically. +What: /sys/kernel/debug/qat_<device>_<BDF>/heartbeat/inject_error +Date: January 2024 +KernelVersion: 6.7 +Contact: [email protected] +Description: (WO) Write to inject an error that simulates an heartbeat + failure. This is to be used for testing purposes. + + After writing this file, the driver stops arbitration on a + random engine and disables the fetching of heartbeat counters. + If a workload is running on the device, a job submitted to the + accelerator might not get a response and a read of the + `heartbeat/status` attribute might report -1, i.e. device + unresponsive. + The error is unrecoverable thus the device must be restarted to + restore its functionality. + + This attribute is available only when the kernel is built with + CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION=y. + + A write of 1 enables error injection. + + The following example shows how to enable error injection:: + + # cd /sys/kernel/debug/qat_<device>_<BDF> + # echo 1 > heartbeat/inject_error + What: /sys/kernel/debug/qat_<device>_<BDF>/pm_status Date: January 2024 KernelVersion: 6.7
drivers/crypto/intel/qat/Kconfig +15 -0
@@ -95,3 +95,18 @@ config CRYPTO_DEV_QAT_C62XVF To compile this as a module, choose M here: the module will be called qat_c62xvf. + +config CRYPTO_DEV_QAT_ERROR_INJECTION + bool "Support for Intel(R) QAT Devices Heartbeat Error Injection" + default n + depends on CRYPTO_DEV_QAT + depends on DEBUG_FS + help + Enables a mechanism that allows to inject a heartbeat error on + Intel(R) QuickAssist devices for testing purposes. + + This is intended for developer use only. + If unsure, say N. + + This functionality is available via debugfs entry of the Intel(R) + QuickAssist device
drivers/crypto/intel/qat/qat_common/Makefile +4 -0
@@ -49,3 +49,7 @@ intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \ adf_gen2_pfvf.o adf_gen4_pfvf.o + +intel_qat-$(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION) += adf_heartbeat_inject.o + +ccflags-$(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION) += -DQAT_HB_ERROR_INJECTION
drivers/crypto/intel/qat/qat_common/adf_common_drv.h +3 -0
@@ -90,6 +90,9 @@ void adf_exit_aer(void);int adf_init_arb (struct adf_accel_dev *accel_dev);void adf_exit_arb (struct adf_accel_dev *accel_dev);void adf_update_ring_arb (struct adf_etr_ring_data *ring); +#ifdef QAT_HB_ERROR_INJECTION +int adf_disable_arb_thd (struct adf_accel_dev *accel_dev,u32 ae,u32 thr); +#endif int adf_dev_get (struct adf_accel_dev *accel_dev);void adf_dev_put (struct adf_accel_dev *accel_dev);
drivers/crypto/intel/qat/qat_common/adf_heartbeat.c +0 -6
@@ -23,12 +23,6 @@#define ADF_HB_EMPTY_SIG 0xA5A5A5A5 -/* Heartbeat counter pair */ -struct hb_cnt_pair { -__u16 resp_heartbeat_cnt; -__u16 req_heartbeat_cnt; -}; -static int adf_hb_check_polling_freq (struct adf_accel_dev *accel_dev) {u64 curr_time =adf_clock_get_current_time ();
drivers/crypto/intel/qat/qat_common/adf_heartbeat.h +12 -0
@@ -19,6 +19,12 @@ enum adf_device_heartbeat_status { HB_DEV_UNSUPPORTED, }; +/* Heartbeat counter pair */ +struct hb_cnt_pair { +__u16 resp_heartbeat_cnt; +__u16 req_heartbeat_cnt; +}; +struct adf_heartbeat {unsigned int hb_sent_counter;unsigned int hb_failed_counter;
@@ -35,6 +41,9 @@ struct adf_heartbeat {struct dentry *cfg;struct dentry *sent;struct dentry *failed; +#ifdef QAT_HB_ERROR_INJECTION +struct dentry *inject_error; +#endif } dbgfs; };
@@ -50,6 +59,9 @@ int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev,void adf_heartbeat_status (struct adf_accel_dev *accel_dev,enum adf_device_heartbeat_status *hb_status);void adf_heartbeat_check_ctrs (struct adf_accel_dev *accel_dev); +#ifdef QAT_HB_ERROR_INJECTION +int adf_heartbeat_inject_error (struct adf_accel_dev *accel_dev); +#endif #else static inline int adf_heartbeat_init (struct adf_accel_dev *accel_dev)
drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c +48 -0
66 lines not on this page · raw message
drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c new file +76 -0
76 lines not on this page · raw message
drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c +34 -0
@@ -103,3 +103,37 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) csr_ops->write_csr_ring_srv_arb_en (csr, i,0 ); }EXPORT_SYMBOL_GPL (adf_exit_arb); + +#ifdef QAT_HB_ERROR_INJECTION +static void adf_write_arb_wt2sam (void __iomem *csr_addr,u32 csr_offset, +u32 wrk_to_ser_map_offset,size_t index,u32 value) +{ +WRITE_CSR_ARB_WT2SAM (csr_addr, csr_offset, wrk_to_ser_map_offset, index, + value); +} + +int adf_disable_arb_thd (struct adf_accel_dev *accel_dev,u32 ae,u32 thr) +{ +void __iomem *csr = accel_dev->transport->banks[0 ].csr_addr; +struct adf_hw_device_data *hw_data = accel_dev->hw_device; +const u32 *thd_2_arb_cfg; +struct arb_info info; +u32 ae_thr_map; + +if (ADF_AE_STRAND0_THREAD == thr ||ADF_AE_STRAND1_THREAD == thr) + thr =ADF_AE_ADMIN_THREAD ; + + hw_data->get_arb_info (&info); + thd_2_arb_cfg = hw_data->get_arb_mapping (accel_dev); +if (!thd_2_arb_cfg) +return -EFAULT ; + +/* Disable scheduling for this particular AE and thread */ + ae_thr_map = *(thd_2_arb_cfg + ae); + ae_thr_map &= ~(0x0F << (thr *4 )); + +adf_write_arb_wt2sam (csr, info.arb_offset, info.wt2sam_offset, ae, + ae_thr_map); +return 0 ; +} +#endif
base-commit: 36c3294e09436a7616dc68c9134ece30bb2d4578
Signature · 1 line
2.41.0
Herbert Xu
maintainerdrivers/crypto/intel/qat/Kconfig config CRYPTO_DEV_QAT_C62XVF · line 101
Quoted 4 lines · Shashank Gupta
drivers/crypto/intel/qat/Kconfig +3
-0+config CRYPTO_DEV_QAT_ERROR_INJECTION + bool "Support for Intel(R) QAT Devices Heartbeat Error Injection" + default n
Please remove this line as n is the default.
Thanks,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Herbert Xu
maintainerQuoted 2 lines · Shashank Gupta
+#ifdef
QAT_HB_ERROR_INJECTION
Please move this into the code.
drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c void adf_heartbeat_dbgfs_add(struct adf_accel_dev *accel_dev) · line 217
Quoted 5 lines
drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c +5
-0+#ifdef QAT_HB_ERROR_INJECTION + hb->dbgfs.inject_error =debugfs_create_file ("inject_error" ,0200 , + hb->dbgfs.base_dir, accel_dev, + &adf_hb_error_inject_fops); +#endif
So this would look like
if (IS_ENABLED(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION)) {
struct dentry *inject_error __maybe_unused;
inject_error = debugfs_create_file("inject_error", 0200,
hb->dbgfs.base_dir, accel_dev,
&adf_hb_error_inject_fops);
#ifdef CONFIG_CRYPTO_DEV_QAT_HB_ERROR_INJECTION
hb->dbgfs.inject_error = inject_error;
#endif
}Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt