accel/habanalabs: add helper to extract the FW major/minor
Subsystems (4)
Oded Gabbay
maintainerPATCH 1/10 accel/habanalabs: add helper to extract the FW major/minor
From: Dafna Hirschfeld <[email protected]>
the helper is extract_u32_until_given_char and can later be used to also get the major/minor of the sw version.
--- drivers/accel/habanalabs/common/firmware_if.c | 69 ++++++++++++------- 1 file changed, 45 insertions(+), 24 deletions(-)
drivers/accel/habanalabs/common/firmware_if.c +45 -24
@@ -71,38 +71,59 @@ static char *extract_fw_ver_from_str(const char *fw_str)return NULL ; } -static int hl_get_preboot_major_minor (struct hl_device *hdev,char *preboot_ver) +/** +* extract_u32_until_given_char() - given a string of the format "<u32><char>*", extract the u32. +* @str: the given string +* @ver_num: the pointer to the extracted u32 to be returned to the caller. +* @given_char: the given char at the end of the u32 in the string +* +* Return: Upon success, return a pointer to the given_char in the string. Upon failure, return NULL +*/ +static char *extract_u32_until_given_char (char *str,u32 *ver_num,char given_char) { -char major[8 ], minor[8 ], *first_dot, *second_dot; -int rc; +char num_str[8 ] = {}, *ch; - first_dot =strnstr (preboot_ver,"." ,10 ); -if (first_dot) { -strscpy (major, preboot_ver, first_dot - preboot_ver +1 ); - rc =kstrtou32 (major,10 , &hdev->fw_major_version); - }else { - rc = -EINVAL ; - } + ch =strchrnul (str, given_char); +if (*ch =='\0' || ch == str || ch - str >=sizeof (num_str)) +return NULL ; -if (rc) { -dev_err (hdev->dev,"Error %d parsing preboot major version\n" , rc); -return rc; +memcpy (num_str, str, ch - str); +if (kstrtou32 (num_str,10 , ver_num)) +return NULL ; +return ch; +} + +/** +* hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string. +* @hdev: pointer to the hl_device +* @preboot_ver: the FW's version string +* +* preboot_ver is expected to be the format of <major>.<minor>.<sub minor>*, e.g: 42.0.1-sec-3 +* The extracted version is set in the hdev fields: fw_inner_{major/minor}_ver. +* +* Return: 0 on success, negative error code for failure. +*/ +static int hl_get_preboot_major_minor (struct hl_device *hdev,char *preboot_ver) +{ + preboot_ver =extract_u32_until_given_char (preboot_ver, &hdev->fw_major_version,'.' ); +if (!preboot_ver) { +dev_err (hdev->dev,"Error parsing preboot major version\n" ); + goto err_zero_ver; } -/* skip the first dot */ - first_dot++; + preboot_ver++; - second_dot =strnstr (first_dot,"." ,10 ); -if (second_dot) { -strscpy (minor, first_dot, second_dot - first_dot +1 ); - rc =kstrtou32 (minor,10 , &hdev->fw_minor_version); - }else { - rc = -EINVAL ; + preboot_ver =extract_u32_until_given_char (preboot_ver, &hdev->fw_minor_version,'.' ); +if (!preboot_ver) { +dev_err (hdev->dev,"Error parsing preboot minor version\n" ); + goto err_zero_ver; } +return 0 ; -if (rc) -dev_err (hdev->dev,"Error %d parsing preboot minor version\n" , rc); -return rc; +err_zero_ver: + hdev->fw_major_version =0 ; + hdev->fw_minor_version =0 ; +return -EINVAL ; } static int hl_request_fw(struct hl_device *hdev,
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 2/10 accel/habanalabs: rename fw_{major/minor}_version to fw_inner_{major/minor}_ver
From: Dafna Hirschfeld <[email protected]>
We later want to add fields for Firmware SW version. The current extracted FW version is the inner FW versioning so the new name is better and also better differentiate from the FW's SW version.
--- drivers/accel/habanalabs/common/firmware_if.c | 8 ++++---- drivers/accel/habanalabs/common/habanalabs.h | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-)
drivers/accel/habanalabs/common/firmware_if.c +4 -4
@@ -105,7 +105,7 @@ static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_ch*/ static int hl_get_preboot_major_minor (struct hl_device *hdev,char *preboot_ver) { - preboot_ver =extract_u32_until_given_char (preboot_ver, &hdev->fw_major_version,'.' ); + preboot_ver =extract_u32_until_given_char (preboot_ver, &hdev->fw_inner_major_ver,'.' );if (!preboot_ver) {dev_err (hdev->dev,"Error parsing preboot major version\n" ); goto err_zero_ver;
@@ -113,7 +113,7 @@ static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver) preboot_ver++; - preboot_ver =extract_u32_until_given_char (preboot_ver, &hdev->fw_minor_version,'.' ); + preboot_ver =extract_u32_until_given_char (preboot_ver, &hdev->fw_inner_minor_ver,'.' );if (!preboot_ver) {dev_err (hdev->dev,"Error parsing preboot minor version\n" ); goto err_zero_ver;
@@ -121,8 +121,8 @@ static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver)return 0 ; err_zero_ver: - hdev->fw_major_version =0 ; - hdev->fw_minor_version =0 ; + hdev->fw_inner_major_ver =0 ; + hdev->fw_inner_minor_ver =0 ;return -EINVAL ; }
drivers/accel/habanalabs/common/habanalabs.h +5 -5
@@ -3225,8 +3225,8 @@ struct hl_reset_info {* @captured_err_info: holds information about errors. * @reset_info: holds current device reset information. * @stream_master_qid_arr: pointer to array with QIDs of master streams. -* @fw_major_version: major version of current loaded preboot. -* @fw_minor_version: minor version of current loaded preboot. +* @fw_inner_major_ver: the major of current loaded preboot inner version. +* @fw_inner_minor_ver: the minor of current loaded preboot inner version. * @dram_used_mem: current DRAM memory consumption. * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram * @timeout_jiffies: device CS timeout value.
@@ -3412,8 +3412,8 @@ struct hl_device {struct hl_reset_info reset_info;u32 *stream_master_qid_arr; -u32 fw_major_version; -u32 fw_minor_version; +u32 fw_inner_major_ver; +u32 fw_inner_minor_ver;atomic64_t dram_used_mem;u64 memory_scrub_val;u64 timeout_jiffies;
@@ -3549,7 +3549,7 @@ struct hl_ioctl_desc {static inline bool hl_is_fw_ver_below_1_9 (struct hl_device *hdev) { -return (hdev->fw_major_version <42 ); +return (hdev->fw_inner_major_ver <42 ); }/*
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 3/10 accel/habanalabs: extract and save the FW's SW major/minor/sub-minor
In reply to Oded GabbayFrom: Dafna Hirschfeld <[email protected]>
It is not always possible to know the FW's SW version from the inner FW version. Therefore we should extract the general SW version in addition to the FW version and use it in functions like 'hl_is_fw_ver_below_1_9' etc.
--- drivers/accel/habanalabs/common/firmware_if.c | 78 +++++++++++++++++-- drivers/accel/habanalabs/common/habanalabs.h | 6 ++ 2 files changed, 78 insertions(+), 6 deletions(-)
drivers/accel/habanalabs/common/firmware_if.c +72 -6
@@ -93,6 +93,71 @@ static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_chreturn ch; } +/** +* hl_get_sw_major_minor_subminor() - extract the FW's SW version major, minor, sub-minor +* from the version string +* @hdev: pointer to the hl_device +* @fw_str: the FW's version string +* +* The extracted version is set in the hdev fields: fw_sw_{major/minor/sub_minor}_ver. +* +* fw_str is expected to have one of two possible formats, examples: +* 1) 'Preboot version hl-gaudi2-1.9.0-fw-42.0.1-sec-3' +* 2) 'Preboot version hl-gaudi2-1.9.0-rc-fw-42.0.1-sec-3' +* In those examples, the SW major,minor,subminor are correspondingly: 1,9,0. +* +* Return: 0 for success or a negative error code for failure. +*/ +static int hl_get_sw_major_minor_subminor (struct hl_device *hdev,const char *fw_str) +{ +char *end, *start; + + end =strnstr (fw_str,"-rc-" ,VERSION_MAX_LEN ); +if (end == fw_str) +return -EINVAL ; + +if (!end) + end =strnstr (fw_str,"-fw-" ,VERSION_MAX_LEN ); + +if (end == fw_str) +return -EINVAL ; + +if (!end) +return -EINVAL ; + +for (start = end -1 ; start != fw_str; start--) { +if (*start =='-' ) +break ; + } + +if (start == fw_str) +return -EINVAL ; + +/* start/end point each to the starting and ending hyphen of the sw version e.g. -1.9.0- */ + start++; + start =extract_u32_until_given_char (start, &hdev->fw_sw_major_ver,'.' ); +if (!start) + goto err_zero_ver; + + start++; + start =extract_u32_until_given_char (start, &hdev->fw_sw_minor_ver,'.' ); +if (!start) + goto err_zero_ver; + + start++; + start =extract_u32_until_given_char (start, &hdev->fw_sw_sub_minor_ver,'-' ); +if (!start) + goto err_zero_ver; + +return 0 ; + +err_zero_ver: + hdev->fw_sw_major_ver =0 ; + hdev->fw_sw_minor_ver =0 ; + hdev->fw_sw_sub_minor_ver =0 ; +return -EINVAL ; +} +/** * hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string. * @hdev: pointer to the hl_device
@@ -2172,6 +2237,7 @@ static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,struct asic_fixed_properties *prop = &hdev->asic_prop;char *preboot_ver, *boot_ver;char btl_ver[32 ]; +int rc;switch (fwc) {case FW_COMP_BOOT_FIT :
@@ -2185,20 +2251,20 @@ static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,break ;case FW_COMP_PREBOOT :strscpy (prop->preboot_ver, fw_version,VERSION_MAX_LEN ); - preboot_ver =strnstr (prop->preboot_ver,"Preboot" , -VERSION_MAX_LEN ); + preboot_ver =strnstr (prop->preboot_ver,"Preboot" ,VERSION_MAX_LEN ); +dev_info (hdev->dev,"preboot full version: '%s'\n" , preboot_ver); +if (preboot_ver && preboot_ver != prop->preboot_ver) {strscpy (btl_ver, prop->preboot_ver,min ((int ) (preboot_ver - prop->preboot_ver),31 ));dev_info (hdev->dev,"%s\n" , btl_ver); } + rc =hl_get_sw_major_minor_subminor (hdev, preboot_ver); +if (rc) +return rc; preboot_ver =extract_fw_ver_from_str (prop->preboot_ver);if (preboot_ver) { -int rc; - -dev_info (hdev->dev,"preboot version %s\n" , preboot_ver); - rc =hl_get_preboot_major_minor (hdev, preboot_ver);kfree (preboot_ver);if (rc)
drivers/accel/habanalabs/common/habanalabs.h +6 -0
@@ -3227,6 +3227,9 @@ struct hl_reset_info {* @stream_master_qid_arr: pointer to array with QIDs of master streams. * @fw_inner_major_ver: the major of current loaded preboot inner version. * @fw_inner_minor_ver: the minor of current loaded preboot inner version. +* @fw_sw_major_ver: the major of current loaded preboot SW version. +* @fw_sw_minor_ver: the minor of current loaded preboot SW version. +* @fw_sw_sub_minor_ver: the sub-minor of current loaded preboot SW version. * @dram_used_mem: current DRAM memory consumption. * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram * @timeout_jiffies: device CS timeout value.
@@ -3414,6 +3417,9 @@ struct hl_device {u32 *stream_master_qid_arr;u32 fw_inner_major_ver;u32 fw_inner_minor_ver; +u32 fw_sw_major_ver; +u32 fw_sw_minor_ver; +u32 fw_sw_sub_minor_ver;atomic64_t dram_used_mem;u64 memory_scrub_val;u64 timeout_jiffies;
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 4/10 accel/habanalabs: check fw version using sw version
In reply to Oded GabbayFrom: Dafna Hirschfeld <[email protected]>
The fw inner version is less trustable, instead use the fw general sw release version.
--- drivers/accel/habanalabs/common/habanalabs.h | 10 ++++++++-- drivers/accel/habanalabs/gaudi2/gaudi2.c | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/accel/habanalabs/common/habanalabs.h +8 -2
@@ -3553,9 +3553,15 @@ struct hl_ioctl_desc {hl_ioctl_t *func; }; -static inline bool hl_is_fw_ver_below_1_9 (struct hl_device *hdev) +static inline bool hl_is_fw_sw_ver_below (struct hl_device *hdev,u32 fw_sw_major,u32 fw_sw_minor) { -return (hdev->fw_inner_major_ver <42 ); +if (hdev->fw_sw_major_ver < fw_sw_major) +return true; +if (hdev->fw_sw_major_ver > fw_sw_major) +return false; +if (hdev->fw_sw_minor_ver < fw_sw_minor) +return true; +return false; }/*
drivers/accel/habanalabs/gaudi2/gaudi2.c +1 -1
@@ -8043,7 +8043,7 @@ static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,case RAZWI_TPC : hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx]; -if (hl_is_fw_ver_below_1_9 (hdev) && +if (hl_is_fw_sw_ver_below (hdev,1 ,9 ) && !hdev->asic_prop.fw_security_enabled && ((module_idx ==0 ) || (module_idx ==1 ))) lbw_rtr_id =DCORE0_RTR0 ;
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 5/10 accel/habanalabs: do soft-reset using cpucp packet
In reply to Oded GabbayFrom: Dafna Hirschfeld <[email protected]>
This is done depending on the FW version. The cpucp method is preferable and saves scratchpads resource.
--- drivers/accel/habanalabs/common/firmware_if.c | 14 ++++++++++ drivers/accel/habanalabs/common/habanalabs.h | 1 + drivers/accel/habanalabs/gaudi2/gaudi2.c | 26 ++++++++++++------- .../habanalabs/include/common/cpucp_if.h | 4 +++ 4 files changed, 35 insertions(+), 10 deletions(-)
drivers/accel/habanalabs/common/firmware_if.c +14 -0
@@ -591,6 +591,20 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, size); } +int hl_fw_send_soft_reset (struct hl_device *hdev) +{ +struct cpucp_packet pkt; +int rc; + +memset (&pkt,0 ,sizeof (pkt)); + pkt.ctl =cpu_to_le32 (CPUCP_PACKET_SOFT_RESET <<CPUCP_PKT_CTL_OPCODE_SHIFT ); + rc = hdev->asic_funcs->send_cpu_message (hdev, (u32 *) &pkt,sizeof (pkt),0 ,NULL ); +if (rc) +dev_err (hdev->dev,"failed to send soft-reset msg (err = %d)\n" , rc); + +return rc; +} +int hl_fw_send_device_activity (struct hl_device *hdev,bool open) {struct cpucp_packet pkt;
drivers/accel/habanalabs/common/habanalabs.h +1 -0
@@ -3884,6 +3884,7 @@ int hl_fw_dram_replaced_row_get(struct hl_device *hdev,int hl_fw_dram_pending_row_get (struct hl_device *hdev,u32 *pend_rows_num);int hl_fw_cpucp_engine_core_asid_set (struct hl_device *hdev,u32 asid);int hl_fw_send_device_activity (struct hl_device *hdev,bool open); +int hl_fw_send_soft_reset (struct hl_device *hdev);int hl_pci_bars_map (struct hl_device *hdev,const char *const name[3 ],bool is_wc[3 ]);int hl_pci_elbi_read (struct hl_device *hdev,u64 addr,u32 *data);
drivers/accel/habanalabs/gaudi2/gaudi2.c +16 -10
@@ -6148,18 +6148,24 @@ static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_perform u32 poll_timeout_us) {struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; +int rc =0 ;if (!driver_performs_reset) { -/* set SP to indicate reset request sent to FW */ -if (dyn_regs->cpu_rst_status) -WREG32 (le32_to_cpu (dyn_regs->cpu_rst_status),CPU_RST_STATUS_NA ); -else -WREG32 (mmCPU_RST_STATUS_TO_HOST,CPU_RST_STATUS_NA ); - -WREG32 (le32_to_cpu (dyn_regs->gic_host_soft_rst_irq), - gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET ].cpu_id); - -return gaudi2_get_soft_rst_done_indication (hdev, poll_timeout_us); +if (hl_is_fw_sw_ver_below (hdev,1 ,10 )) { +/* set SP to indicate reset request sent to FW */ +if (dyn_regs->cpu_rst_status) +WREG32 (le32_to_cpu (dyn_regs->cpu_rst_status),CPU_RST_STATUS_NA ); +else +WREG32 (mmCPU_RST_STATUS_TO_HOST,CPU_RST_STATUS_NA ); +WREG32 (le32_to_cpu (dyn_regs->gic_host_soft_rst_irq), + gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET ].cpu_id); + +/* wait for f/w response */ + rc =gaudi2_get_soft_rst_done_indication (hdev, poll_timeout_us); + }else { + rc =hl_fw_send_soft_reset (hdev); + } +return rc; }/* Block access to engines, QMANs and SM during reset, these
drivers/accel/habanalabs/include/common/cpucp_if.h +4 -0
@@ -665,6 +665,9 @@ enum pq_init_status {* * CPUCP_PACKET_REGISTER_INTERRUPTS - * Packet to register interrupts indicating LKD is ready to receive events from FW. +* +* CPUCP_PACKET_SOFT_RESET - +* Packet to perform soft-reset. */ enum cpucp_packet_id {
@@ -731,6 +734,7 @@ enum cpucp_packet_id {CPUCP_PACKET_RESERVED11 ,/* not used */ CPUCP_PACKET_RESERVED12 ,/* internal */ CPUCP_PACKET_REGISTER_INTERRUPTS ,/* internal */ +CPUCP_PACKET_SOFT_RESET ,/* internal */ CPUCP_PACKET_ID_MAX /* must be last */ };
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 6/10 accel/habanalabs: unsecure TPC bias registers
In reply to Oded GabbayFrom: Ofir Bitton <[email protected]>
User needs to be able to perform downcast / upcast of fp8_143 dtype. Hence bias register needs to be accessed by the user.
--- drivers/accel/habanalabs/gaudi2/gaudi2_security.c | 1 + 1 file changed, 1 insertion(+)
drivers/accel/habanalabs/gaudi2/gaudi2_security.c +1 -0
@@ -1542,6 +1542,7 @@ static const u32 gaudi2_pb_dcr0_tpc0_unsecured_regs[] = { mmDCORE0_TPC0_CFG_LUT_FUNC128_BASE2_ADDR_HI, mmDCORE0_TPC0_CFG_LUT_FUNC256_BASE2_ADDR_LO, mmDCORE0_TPC0_CFG_LUT_FUNC256_BASE2_ADDR_HI, + mmDCORE0_TPC0_CFG_FP8_143_BIAS, mmDCORE0_TPC0_CFG_ROUND_CSR, mmDCORE0_TPC0_CFG_CONV_ROUND_CSR, mmDCORE0_TPC0_CFG_SEMAPHORE,
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 7/10 accel/habanalabs: call to HW/FW err returns 0 when no events exist
In reply to Oded GabbayFrom: Moti Haimovski <[email protected]>
This commit modifies the call to retrieve HW or FW error events to return success when no events are pending, as done in the calls to other events.
--- .../habanalabs/common/habanalabs_ioctl.c | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
drivers/accel/habanalabs/common/habanalabs_ioctl.c +10 -10
@@ -842,15 +842,15 @@ static int hw_err_info(struct hl_fpriv *hpriv, struct hl_info_args *args)struct hw_err_info *info;int rc; -if ((!user_buf_size) || (!user_buf)) +if (!user_buf)return -EINVAL ; -if (user_buf_size <sizeof (struct hl_info_hw_err_event )) -return -ENOMEM ; - info = &hdev->captured_err_info.hw_err;if (!info->event_info_available) -return -ENOENT ; +return 0 ; + +if (user_buf_size <sizeof (struct hl_info_hw_err_event )) +return -ENOMEM ; rc =copy_to_user (user_buf, &info->event,sizeof (struct hl_info_hw_err_event ));return rc ? -EFAULT :0 ;
@@ -864,15 +864,15 @@ static int fw_err_info(struct hl_fpriv *hpriv, struct hl_info_args *args)struct fw_err_info *info;int rc; -if ((!user_buf_size) || (!user_buf)) +if (!user_buf)return -EINVAL ; -if (user_buf_size <sizeof (struct hl_info_fw_err_event )) -return -ENOMEM ; - info = &hdev->captured_err_info.fw_err;if (!info->event_info_available) -return -ENOENT ; +return 0 ; + +if (user_buf_size <sizeof (struct hl_info_fw_err_event )) +return -ENOMEM ; rc =copy_to_user (user_buf, &info->event,sizeof (struct hl_info_fw_err_event ));return rc ? -EFAULT :0 ;
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 8/10 accel/habanalabs: add unregister timestamp uapi
In reply to Oded GabbayFrom: farah kassabri <[email protected]>
Add uapi to allow user to unregister timestamp record. This is needed when the user wishes to re-use the same record with different interrupt id. For that, the user must first unregister it from the current interrupt id and then register it with the new id.
--- .../habanalabs/common/command_submission.c | 123 ++++++++++++++---- include/uapi/drm/habanalabs_accel.h | 1 + 2 files changed, 101 insertions(+), 23 deletions(-)
drivers/accel/habanalabs/common/command_submission.c +100 -23
@@ -3196,34 +3196,57 @@ static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)return 0 ; } +static inline void set_record_cq_info (struct hl_user_pending_interrupt *record, +struct hl_cb *cq_cb,u32 cq_offset,u32 target_value) +{ + record->ts_reg_info.cq_cb = cq_cb; + record->cq_kernel_addr = (u64 *) cq_cb->kernel_address + cq_offset; + record->cq_target_value = target_value; +} + +static int validate_and_get_ts_record (struct device *dev, +struct hl_ts_buff *ts_buff,u64 ts_offset, +struct hl_user_pending_interrupt **req_event_record) +{ +struct hl_user_pending_interrupt *ts_cb_last; + + *req_event_record = (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + + ts_offset; + ts_cb_last = (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + + (ts_buff->kernel_buff_size /sizeof (struct hl_user_pending_interrupt )); + +/* Validate ts_offset not exceeding last max */ +if (*req_event_record >= ts_cb_last) { +dev_err (dev,"Ts offset(%llx) exceeds max CB offset(0x%llx)\n" , + ts_offset, (u64 )(uintptr_t )ts_cb_last); +return -EINVAL ; + } + +return 0 ; +} +static int ts_buff_get_kernel_ts_record (struct hl_mmap_mem_buf *buf,struct hl_cb *cq_cb,u64 ts_offset,u64 cq_offset,u64 target_value,spinlock_t *wait_list_lock,struct hl_user_pending_interrupt **pend) { +struct hl_user_pending_interrupt *requested_offset_record;struct hl_ts_buff *ts_buff = buf->private; -struct hl_user_pending_interrupt *requested_offset_record = - (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + - ts_offset; -struct hl_user_pending_interrupt *cb_last = - (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + - (ts_buff->kernel_buff_size /sizeof (struct hl_user_pending_interrupt )); -unsigned long iter_counter =0 ; +unsigned long iter_counter =0 , flags;u64 current_cq_counter;ktime_t timestamp; +int rc; -/* Validate ts_offset not exceeding last max */ -if (requested_offset_record >= cb_last) { -dev_err (buf->mmg->dev,"Ts offset exceeds max CB offset(0x%llx)\n" , - (u64 )(uintptr_t )cb_last); -return -EINVAL ; - } + rc =validate_and_get_ts_record (buf->mmg->dev, ts_buff, ts_offset, + &requested_offset_record); +if (rc) +return rc; timestamp =ktime_get (); start_over: -spin_lock (wait_list_lock); +spin_lock_irqsave (wait_list_lock, flags);/* Unregister only if we didn't reach the target value * since in this case there will be no handling in irq context
@@ -3234,7 +3257,9 @@ static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf, current_cq_counter = *requested_offset_record->cq_kernel_addr;if (current_cq_counter < requested_offset_record->cq_target_value) {list_del (&requested_offset_record->wait_list_node); -spin_unlock (wait_list_lock); +spin_unlock_irqrestore (wait_list_lock, flags); + +set_record_cq_info (requested_offset_record, cq_cb, cq_offset, target_value);hl_mmap_mem_buf_put (requested_offset_record->ts_reg_info.buf);hl_cb_put (requested_offset_record->ts_reg_info.cq_cb);
@@ -3245,8 +3270,8 @@ static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf,dev_dbg (buf->mmg->dev,"ts node in middle of irq handling\n" ); -/* irq thread handling in the middle give it time to finish */ -spin_unlock (wait_list_lock); +/* irq handling in the middle give it time to finish */ +spin_unlock_irqrestore (wait_list_lock, flags);usleep_range (100 ,1000 );if (++iter_counter ==MAX_TS_ITER_NUM ) { dev_err(buf->mmg->dev,
@@ -3260,14 +3285,11 @@ static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf, }else {/* Fill up the new registration node info */ requested_offset_record->ts_reg_info.buf = buf; - requested_offset_record->ts_reg_info.cq_cb = cq_cb; requested_offset_record->ts_reg_info.timestamp_kernel_addr = (u64 *) ts_buff->user_buff_address + ts_offset; - requested_offset_record->cq_kernel_addr = - (u64 *) cq_cb->kernel_address + cq_offset; - requested_offset_record->cq_target_value = target_value; +set_record_cq_info (requested_offset_record, cq_cb, cq_offset, target_value); -spin_unlock (wait_list_lock); +spin_unlock_irqrestore (wait_list_lock, flags); } *pend = requested_offset_record;
@@ -3277,6 +3299,58 @@ static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf,return 0 ; } +static int unregister_timestamp_node_ioctl (struct hl_device *hdev,struct hl_mem_mgr *mmg, +u64 ts_handle,u64 ts_offset,struct hl_user_interrupt *interrupt) +{ +struct hl_user_pending_interrupt *req_event_record, *pend, *temp_pend; +struct hl_mmap_mem_buf *buff; +struct hl_ts_buff *ts_buff; +bool ts_rec_found = false; +int rc; + + buff =hl_mmap_mem_buf_get (mmg, ts_handle); +if (!buff) { +dev_err (hdev->dev,"invalid TS buff handle!\n" ); +return -EINVAL ; + } + + ts_buff = buff->private; + + rc =validate_and_get_ts_record (hdev->dev, ts_buff, ts_offset, &req_event_record); +if (rc) + goto out; + +/* +* Note: we don't use the ts in_use field here, but we rather scan the list +* because we cannot rely on the user to keep the order of register/unregister calls +* and since we might have races here all the time between the irq and register/unregister +* calls so it safer to lock the list and scan it to find the node. +* If the node found on the list we mark it as not in use and delete it from the list, +* if it's not here then the node was handled already in the irq before we get into +* this ioctl. +*/ +spin_lock (&interrupt->wait_list_lock); +list_for_each_entry_safe (pend, temp_pend, &interrupt->wait_list_head, wait_list_node) { +if (pend == req_event_record) { + pend->ts_reg_info.in_use =0 ; +list_del (&pend->wait_list_node); + ts_rec_found = true; +break ; + } + } +spin_unlock (&interrupt->wait_list_lock); + +/* Put refcounts that were taken when we registered the event */ +if (ts_rec_found) { +hl_mmap_mem_buf_put (pend->ts_reg_info.buf); +hl_cb_put (pend->ts_reg_info.cq_cb); + } +out: +hl_mmap_mem_buf_put (buff); + +return rc; +} + static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_mem_mgr *cb_mmg, struct hl_mem_mgr *mmg, u64 timeout_us, u64 cq_counters_handle, u64 cq_counters_offset,
@@ -3610,7 +3684,10 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data)return -EINVAL ; } -if (args->in.flags &HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ ) +if (args->in.flags &HL_WAIT_CS_FLAGS_UNREGISTER_INTERRUPT ) + rc =unregister_timestamp_node_ioctl (hdev, &hpriv->mem_mgr, + args->in.timestamp_handle, args->in.timestamp_offset, interrupt); +else if (args->in.flags &HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ ) rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx, &hpriv->mem_mgr, &hpriv->mem_mgr, args->in.interrupt_timeout_us, args->in.cq_counters_handle, args->in.cq_counters_offset,
include/uapi/drm/habanalabs_accel.h +1 -0
@@ -1704,6 +1704,7 @@ union hl_cs_args {#define HL_WAIT_CS_FLAGS_MULTI_CS 0x4#define HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ 0x10#define HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT 0x20 +#define HL_WAIT_CS_FLAGS_UNREGISTER_INTERRUPT 0x40#define HL_WAIT_MULTI_CS_LIST_MAX_LEN 32
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 9/10 accel/habanalabs: minimize encapsulation signal mutex lock time
In reply to Oded GabbayFrom: Koby Elbaz <[email protected]>
Sync Stream Encapsulated Signal Handlers can be managed from different contexts, and as such they are protected via a spin_lock. However, spin_lock was unnecessarily protecting a larger code section than really needed, covering a sleepable code section as well. Since spin_lock disables preemption, it could lead to sleeping in atomic context.
--- drivers/accel/habanalabs/common/command_submission.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
drivers/accel/habanalabs/common/command_submission.c +8 -2
@@ -2152,7 +2152,7 @@ static int cs_ioctl_unreserve_signals(struct hl_fpriv *hpriv, u32 handle_id) hdev->asic_funcs->hw_queues_unlock (hdev); rc = -EINVAL ; - goto out; + goto out_unlock; }/*
@@ -2167,15 +2167,21 @@ static int cs_ioctl_unreserve_signals(struct hl_fpriv *hpriv, u32 handle_id)/* Release the id and free allocated memory of the handle */ idr_remove (&mgr->handles, handle_id); + +/* unlock before calling ctx_put, where we might sleep */ +spin_unlock (&mgr->lock);hl_ctx_put (encaps_sig_hdl->ctx);kfree (encaps_sig_hdl); + goto out; }else { rc = -EINVAL ;dev_err (hdev->dev,"failed to unreserve signals, cannot find handler\n" ); } -out: + +out_unlock:spin_unlock (&mgr->lock); +out:return rc; }
Signature · 1 line
2.40.0
Oded Gabbay
maintainerPATCH 10/10 accel/habanalabs: refactor abort of completions and waits
In reply to Oded GabbayFrom: Koby Elbaz <[email protected]>
Aborting CS completions should be in command_submission.c but aborting waiting for user interrupts should be in device.c.
This separation is also for adding more abort operations in the future.
--- .../habanalabs/common/command_submission.c | 3 +-- drivers/accel/habanalabs/common/device.c | 17 ++++++++++++----- drivers/accel/habanalabs/common/habanalabs.h | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-)
drivers/accel/habanalabs/common/command_submission.c +1 -2
@@ -1139,11 +1139,10 @@ static void force_complete_cs(struct hl_device *hdev)spin_unlock (&hdev->cs_mirror_lock); } -void hl_abort_waitings_for_completion (struct hl_device *hdev) +void hl_abort_waiting_for_cs_completions (struct hl_device *hdev) {force_complete_cs (hdev);force_complete_multi_cs (hdev); -hl_release_pending_user_interrupts (hdev); }static void job_wq_completion (struct work_struct *work)
drivers/accel/habanalabs/common/device.c +12 -5
@@ -1157,6 +1157,16 @@ static void take_release_locks(struct hl_device *hdev)mutex_unlock (&hdev->fpriv_ctrl_list_lock); } +static void hl_abort_waiting_for_completions (struct hl_device *hdev) +{ +hl_abort_waiting_for_cs_completions (hdev); + +/* Release all pending user interrupts, each pending user interrupt +* holds a reference to a user context. +*/ +hl_release_pending_user_interrupts (hdev); +} +static void cleanup_resources (struct hl_device *hdev,bool hard_reset,bool fw_reset,bool skip_wq_flush) {
@@ -1176,10 +1186,7 @@ static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_r/* flush the MMU prefetch workqueue */ flush_workqueue (hdev->prefetch_wq); -/* Release all pending user interrupts, each pending user interrupt -* holds a reference to user context -*/ -hl_release_pending_user_interrupts (hdev); +hl_abort_waiting_for_completions (hdev); }/*
@@ -1921,7 +1928,7 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)hl_ctx_put (ctx); -hl_abort_waitings_for_completion (hdev); +hl_abort_waiting_for_completions (hdev);return 0 ;
drivers/accel/habanalabs/common/habanalabs.h +1 -1
@@ -3934,7 +3934,7 @@ void hl_dec_fini(struct hl_device *hdev);void hl_dec_ctx_fini (struct hl_ctx *ctx);void hl_release_pending_user_interrupts (struct hl_device *hdev); -void hl_abort_waitings_for_completion (struct hl_device *hdev); +void hl_abort_waiting_for_cs_completions (struct hl_device *hdev);int hl_cs_signal_sob_wraparound_handler (struct hl_device *hdev,u32 q_idx,struct hl_hw_sob **hw_sob,u32 count,bool encaps_sig);
Signature · 1 line
2.40.0