habanalabs: Timestamps buffers registration
PATCH 1/2 habanalabs: Timestamps buffers registration
From: farah kassabri <[email protected]>
Timestamp registration API allows the user to register a timestamp record event which will make the driver set timestamp when CQ counter reaches the target value and write it to a specific location specified
by the user.
This is a non blocking API, unlike the wait_for_interrupt which is a blocking one.
Signed-off-by farah kassabri <[email protected]>
Reviewed-by Oded Gabbay <[email protected]>
Signed-off-by Oded Gabbay <[email protected]>
--- .../habanalabs/common/command_submission.c | 197 ++++++++++--- drivers/misc/habanalabs/common/device.c | 18 +- drivers/misc/habanalabs/common/habanalabs.h | 106 ++++++- .../misc/habanalabs/common/habanalabs_drv.c | 2 + drivers/misc/habanalabs/common/irq.c | 127 ++++++++- drivers/misc/habanalabs/common/memory.c | 261 ++++++++++++++++++ include/uapi/misc/habanalabs.h | 37 ++- 7 files changed, 690 insertions(+), 58 deletions(-)
drivers/misc/habanalabs/common/command_submission.c +163 -34
@@ -14,6 +14,8 @@#define HL_CS_FLAGS_TYPE_MASK (HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT | \ HL_CS_FLAGS_COLLECTIVE_WAIT) +#define MAX_TS_ITER_NUM 10 +/** * enum hl_cs_wait_status - cs wait status * @CS_WAIT_STATUS_BUSY: cs was not completed yet
@@ -924,7 +926,7 @@ void hl_cs_rollback_all(struct hl_device *hdev)int i;struct hl_cs *cs, *tmp; -flush_workqueue (hdev->sob_reset_wq); +flush_workqueue (hdev->ts_free_obj_wq);/* flush all completions before iterating over the CS mirror list in * order to avoid a race with the release functions
@@ -948,13 +950,19 @@ void hl_cs_rollback_all(struct hl_device *hdev)static void wake_pending_user_interrupt_threads (struct hl_user_interrupt *interrupt) { -struct hl_user_pending_interrupt *pend; +struct hl_user_pending_interrupt *pend, *temp;unsigned long flags;spin_lock_irqsave (&interrupt->wait_list_lock, flags); -list_for_each_entry (pend, &interrupt->wait_list_head, wait_list_node) { - pend->fence.error = -EIO ; -complete_all (&pend->fence.completion); +list_for_each_entry_safe (pend, temp, &interrupt->wait_list_head, wait_list_node) { +if (pend->ts_reg_info.ts_buff) { +list_del (&pend->wait_list_node); +hl_ts_put (pend->ts_reg_info.ts_buff); +hl_cb_put (pend->ts_reg_info.cq_cb); + }else { + pend->fence.error = -EIO ; +complete_all (&pend->fence.completion); + } }spin_unlock_irqrestore (&interrupt->wait_list_lock, flags); }
@@ -2857,43 +2865,133 @@ static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)return 0 ; } +static int ts_buff_get_kernel_ts_record (struct hl_ts_buff *ts_buff, +struct hl_cb *cq_cb, +u64 ts_offset,u64 cq_offset,u64 target_value, +spinlock_t *wait_list_lock, +struct hl_user_pending_interrupt **pend) +{ +struct hl_user_pending_interrupt *requested_offset_record = + (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + + ts_offset; +struct hl_user_pending_interrupt *cb_last = + (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + + (ts_buff->kernel_buff_size /sizeof (struct hl_user_pending_interrupt )); +unsigned long flags, iter_counter =0 ; +u64 current_cq_counter; + +/* Validate ts_offset not exceeding last max */ +if (requested_offset_record > cb_last) { +dev_err (ts_buff->hdev->dev,"Ts offset exceeds max CB offset(0x%llx)\n" , + (u64 )(uintptr_t )cb_last); +return -EINVAL ; + } + +start_over: +spin_lock_irqsave (wait_list_lock, flags); + +/* Unregister only if we didn't reach the target value +* since in this case there will be no handling in irq context +* and then it's safe to delete the node out of the interrupt list +* then re-use it on other interrupt +*/ +if (requested_offset_record->ts_reg_info.in_use) { + current_cq_counter = *requested_offset_record->cq_kernel_addr; +if (current_cq_counter < requested_offset_record->cq_target_value) { +list_del (&requested_offset_record->wait_list_node); +spin_unlock_irqrestore (wait_list_lock, flags); + +hl_ts_put (requested_offset_record->ts_reg_info.ts_buff); +hl_cb_put (requested_offset_record->ts_reg_info.cq_cb); + +dev_dbg (ts_buff->hdev->dev,"ts node removed from interrupt list now can re-use\n" ); + }else { +dev_dbg (ts_buff->hdev->dev,"ts node in middle of irq handling\n" ); + +/* irq handling in the middle give it time to finish */ +spin_unlock_irqrestore (wait_list_lock, flags); +usleep_range (1 ,10 ); +if (++iter_counter ==MAX_TS_ITER_NUM ) { +dev_err (ts_buff->hdev->dev,"handling registration interrupt took too long!!\n" ); +return -EINVAL ; + } + + goto start_over; + } + }else { +spin_unlock_irqrestore (wait_list_lock, flags); + } + +/* Fill up the new registration node info */ + requested_offset_record->ts_reg_info.in_use =1 ; + requested_offset_record->ts_reg_info.ts_buff = ts_buff; + requested_offset_record->ts_reg_info.cq_cb = cq_cb; + requested_offset_record->ts_reg_info.timestamp_kernel_addr = + (u64 *) ts_buff->user_buff_address + ts_offset; + requested_offset_record->cq_kernel_addr = + (u64 *) cq_cb->kernel_address + cq_offset; + requested_offset_record->cq_target_value = target_value; + + *pend = requested_offset_record; + +dev_dbg (ts_buff->hdev->dev,"Found available node in TS kernel CB(0x%llx)\n" , + (u64 )(uintptr_t )requested_offset_record); +return 0 ; +} +static int _hl_interrupt_wait_ioctl (struct hl_device *hdev,struct hl_ctx *ctx, -struct hl_cb_mgr *cb_mgr,u64 timeout_us, -u64 cq_counters_handle,u64 cq_counters_offset, +struct hl_cb_mgr *cb_mgr,struct hl_ts_mgr *ts_mgr, +u64 timeout_us,u64 cq_counters_handle,u64 cq_counters_offset,u64 target_value,struct hl_user_interrupt *interrupt, +bool register_ts_record,u64 ts_handle,u64 ts_offset,u32 *status,u64 *timestamp) { +u32 cq_patched_handle, ts_patched_handle;struct hl_user_pending_interrupt *pend; +struct hl_ts_buff *ts_buff; +struct hl_cb *cq_cb;unsigned long timeout, flags;long completion_rc; -struct hl_cb *cb;int rc =0 ; -u32 handle; timeout =hl_usecs64_to_jiffies (timeout_us);hl_ctx_get (hdev, ctx); - cq_counters_handle >>=PAGE_SHIFT ; - handle = (u32 ) cq_counters_handle; - - cb =hl_cb_get (hdev, cb_mgr, handle); -if (!cb) { -hl_ctx_put (ctx); -return -EINVAL ; + cq_patched_handle =lower_32_bits (cq_counters_handle >>PAGE_SHIFT ); + cq_cb =hl_cb_get (hdev, cb_mgr, cq_patched_handle); +if (!cq_cb) { + rc = -EINVAL ; + goto put_ctx; } - pend =kzalloc (sizeof (*pend),GFP_KERNEL ); -if (!pend) { -hl_cb_put (cb); -hl_ctx_put (ctx); -return -ENOMEM ; - } +if (register_ts_record) { +dev_dbg (hdev->dev,"Timestamp registration: interrupt id: %u, ts offset: %llu, cq_offset: %llu\n" , + interrupt->interrupt_id, ts_offset, cq_counters_offset); -hl_fence_init (&pend->fence,ULONG_MAX ); + ts_patched_handle =lower_32_bits (ts_handle >>PAGE_SHIFT ); + ts_buff =hl_ts_get (hdev, ts_mgr, ts_patched_handle); +if (!ts_buff) { + rc = -EINVAL ; + goto put_cq_cb; + } - pend->cq_kernel_addr = (u64 *) cb->kernel_address + cq_counters_offset; - pend->cq_target_value = target_value; +/* Find first available record */ + rc =ts_buff_get_kernel_ts_record (ts_buff, cq_cb, ts_offset, + cq_counters_offset, target_value, + &interrupt->wait_list_lock, &pend); +if (rc) + goto put_ts_buff; + }else { + pend =kzalloc (sizeof (*pend),GFP_KERNEL ); +if (!pend) { + rc = -ENOMEM ; + goto put_cq_cb; + } +hl_fence_init (&pend->fence,ULONG_MAX ); + pend->cq_kernel_addr = (u64 *) cq_cb->kernel_address + cq_counters_offset; + pend->cq_target_value = target_value; + }spin_lock_irqsave (&interrupt->wait_list_lock, flags);
@@ -2901,13 +2999,19 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,* before we added the node to the wait list */ if (*pend->cq_kernel_addr >= target_value) { +if (register_ts_record) + pend->ts_reg_info.in_use =0 ;spin_unlock_irqrestore (&interrupt->wait_list_lock, flags); *status =HL_WAIT_CS_STATUS_COMPLETED ; -/* There was no interrupt, we assume the completion is now. */ - pend->fence.timestamp =ktime_get (); - goto set_timestamp; +if (register_ts_record) { + *pend->ts_reg_info.timestamp_kernel_addr =ktime_get_ns (); + goto put_ts_buff; + }else { + pend->fence.timestamp =ktime_get (); + goto set_timestamp; + } }else if (!timeout_us) {spin_unlock_irqrestore (&interrupt->wait_list_lock, flags); *status =HL_WAIT_CS_STATUS_BUSY ;
@@ -2916,11 +3020,19 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, }/* Add pending user interrupt to relevant list for the interrupt -* handler to monitor +* handler to monitor. +* Note that we cannot have sorted list by target value, +* in order to shorten the list pass loop, since +* same list could have nodes for different cq counter handle. */ list_add_tail (&pend->wait_list_node, &interrupt->wait_list_head);spin_unlock_irqrestore (&interrupt->wait_list_lock, flags); +if (register_ts_record) { + rc = *status =HL_WAIT_CS_STATUS_COMPLETED ; + goto ts_registration_exit; + } +/* Wait for interrupt handler to signal completion */ completion_rc =wait_for_completion_interruptible_timeout (&pend->fence.completion, timeout);
@@ -2952,15 +3064,30 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, } } +/* +* We keep removing the node from list here, and not at the irq handler +* for completion timeout case. and if it's a registration +* for ts record, the node will be deleted in the irq handler after +* we reach the target value. +*/ spin_lock_irqsave (&interrupt->wait_list_lock, flags);list_del (&pend->wait_list_node);spin_unlock_irqrestore (&interrupt->wait_list_lock, flags); set_timestamp: *timestamp =ktime_to_ns (pend->fence.timestamp); -kfree (pend); -hl_cb_put (cb); +hl_cb_put (cq_cb); +ts_registration_exit: +hl_ctx_put (ctx); + +return rc; + +put_ts_buff: +hl_ts_put (ts_buff); +put_cq_cb: +hl_cb_put (cq_cb); +put_ctx:hl_ctx_put (ctx);return rc;
@@ -3119,11 +3246,13 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data) interrupt = &hdev->user_interrupt[interrupt_id - first_interrupt];if (args->in.flags &HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ ) - rc =_hl_interrupt_wait_ioctl (hdev, hpriv->ctx, &hpriv->cb_mgr, + rc =_hl_interrupt_wait_ioctl (hdev, hpriv->ctx, &hpriv->cb_mgr, &hpriv->ts_mem_mgr, args->in.interrupt_timeout_us, args->in.cq_counters_handle, args->in.cq_counters_offset, - args->in.target, interrupt, &status, - ×tamp); + args->in.target, interrupt, + !!(args->in.flags &HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT ), + args->in.timestamp_handle, args->in.timestamp_offset, + &status, ×tamp);else rc = _hl_interrupt_wait_ioctl_user_addr(hdev, hpriv->ctx, args->in.interrupt_timeout_us, args->in.addr,
drivers/misc/habanalabs/common/device.c +11 -7
@@ -145,6 +145,7 @@ static int hl_device_release(struct inode *inode, struct file *filp)hl_release_pending_user_interrupts (hpriv->hdev);hl_cb_mgr_fini (hdev, &hpriv->cb_mgr); +hl_ts_mgr_fini (hpriv->hdev, &hpriv->ts_mem_mgr);hl_ctx_mgr_fini (hdev, &hpriv->ctx_mgr);if (!hl_hpriv_put (hpriv))
@@ -209,6 +210,9 @@ static int hl_mmap(struct file *filp, struct vm_area_struct *vma)case HL_MMAP_TYPE_BLOCK :return hl_hw_block_mmap (hpriv, vma); + +case HL_MMAP_TYPE_TS_BUFF : +return hl_ts_mmap (hpriv, vma); }return -EINVAL ;
@@ -410,10 +414,10 @@ static int device_early_init(struct hl_device *hdev) goto free_cq_wq; } - hdev->sob_reset_wq =alloc_workqueue ("hl-sob-reset" ,WQ_UNBOUND ,0 ); -if (!hdev->sob_reset_wq) { + hdev->ts_free_obj_wq =alloc_workqueue ("hl-ts-free-obj" ,WQ_UNBOUND ,0 ); +if (!hdev->ts_free_obj_wq) {dev_err (hdev->dev, -"Failed to allocate SOB reset workqueue\n" ); +"Failed to allocate Timestamp registration free workqueue\n" ); rc = -ENOMEM ; goto free_eq_wq; }
@@ -422,7 +426,7 @@ static int device_early_init(struct hl_device *hdev) GFP_KERNEL);if (!hdev->hl_chip_info) { rc = -ENOMEM ; - goto free_sob_reset_wq; + goto free_ts_free_wq; } rc =hl_mmu_if_set_funcs (hdev);
@@ -461,8 +465,8 @@ static int device_early_init(struct hl_device *hdev)hl_cb_mgr_fini (hdev, &hdev->kernel_cb_mgr); free_chip_info:kfree (hdev->hl_chip_info); -free_sob_reset_wq: -destroy_workqueue (hdev->sob_reset_wq); +free_ts_free_wq: +destroy_workqueue (hdev->ts_free_obj_wq); free_eq_wq:destroy_workqueue (hdev->eq_wq); free_cq_wq:
@@ -501,7 +505,7 @@ static void device_early_fini(struct hl_device *hdev)kfree (hdev->hl_chip_info); -destroy_workqueue (hdev->sob_reset_wq); +destroy_workqueue (hdev->ts_free_obj_wq);destroy_workqueue (hdev->eq_wq);destroy_workqueue (hdev->device_reset_work.wq);
drivers/misc/habanalabs/common/habanalabs.h +97 -9
168 lines not on this page · raw message
drivers/misc/habanalabs/common/habanalabs_drv.c +2 -0
@@ -140,6 +140,7 @@ int hl_device_open(struct inode *inode, struct file *filp)hl_cb_mgr_init (&hpriv->cb_mgr);hl_ctx_mgr_init (&hpriv->ctx_mgr); +hl_ts_mgr_init (&hpriv->ts_mem_mgr); hpriv->taskpid =get_task_pid (current,PIDTYPE_PID );
@@ -184,6 +185,7 @@ int hl_device_open(struct inode *inode, struct file *filp) out_err:mutex_unlock (&hdev->fpriv_list_lock);hl_cb_mgr_fini (hpriv->hdev, &hpriv->cb_mgr); +hl_ts_mgr_fini (hpriv->hdev, &hpriv->ts_mem_mgr);hl_ctx_mgr_fini (hpriv->hdev, &hpriv->ctx_mgr); filp->private_data =NULL ;mutex_destroy (&hpriv->restore_phase_mutex);
drivers/misc/habanalabs/common/irq.c +121 -6
143 lines not on this page · raw message
drivers/misc/habanalabs/common/memory.c +261 -0
285 lines not on this page · raw message
include/uapi/misc/habanalabs.h +35 -2
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note * -* Copyright 2016-2020 HabanaLabs, Ltd. +* Copyright 2016-2021 HabanaLabs, Ltd. * All Rights Reserved. * */
@@ -30,6 +30,9 @@*/ #define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 72 +/* Max number of elements in timestamps registration buffers */ +#define TS_MAX_ELEMENTS_NUM (1 << 20)/* 1MB */ +/* * Goya queue Numbering *
@@ -695,10 +698,12 @@ struct hl_cb_in {__u64 cb_handle;/* HL_CB_OP_* */ __u32 op; +/* Size of CB. Maximum size is HL_MAX_CB_SIZE. The minimum size that * will be allocated, regardless of this parameter's value, is PAGE_SIZE */ __u32 cb_size; +/* Context ID - Currently not in use */ __u32 ctx_id;/* HL_CB_FLAGS_* */
@@ -964,6 +969,7 @@ union hl_cs_args {#define HL_WAIT_CS_FLAGS_INTERRUPT_MASK 0xFFF00000#define HL_WAIT_CS_FLAGS_MULTI_CS 0x4#define HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ 0x10 +#define HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT 0x20#define HL_WAIT_MULTI_CS_LIST_MAX_LEN 32
@@ -1036,6 +1042,20 @@ struct hl_wait_cs_in {* relevant only when HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ is set */ __u64 cq_counters_offset; + +/* +* Timestamp_handle timestamps buffer handle. +* relevant only when HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT is set +*/ +__u64 timestamp_handle; + +/* +* Timestamp_offset is offset inside the timestamp buffer pointed by timestamp_handle above. +* upon interrupt, if the cq reached the target value then driver will write +* timestamp to this offset. +* relevant only when HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT is set +*/ +__u64 timestamp_offset; };#define HL_WAIT_CS_STATUS_COMPLETED 0
@@ -1082,6 +1102,14 @@ union hl_wait_cs_args {*/ #define HL_MEM_OP_EXPORT_DMABUF_FD 5 +/* Opcode to create timestamps pool for user interrupts registration support +* The memory will be allocated by the kernel driver, A timestamp buffer which the user +* will get handle to it for mmap, and another internal buffer used by the +* driver for registration management +* The memory will be freed when the user closes the file descriptor(ctx close) +*/ +#define HL_MEM_OP_TS_ALLOC 6 +/* Memory flags */ #define HL_MEM_CONTIGUOUS 0x1#define HL_MEM_SHARED 0x2
@@ -1173,9 +1201,14 @@ struct hl_mem_in {* DMA-BUF file/FD flags. */ __u32 flags; +/* Context ID - Currently not in use */ __u32 ctx_id; -__u32 pad; + +/* number of timestamp elements +* used only when HL_MEM_OP_TS_ALLOC opcode +*/ +__u32 num_of_elements; };struct hl_mem_out {
Signature · 1 line
2.25.1
PATCH 2/2 habanalabs: fix spelling mistake
The name of the property is hints_range_reservation
Signed-off-by Oded Gabbay <[email protected]>
--- drivers/misc/habanalabs/common/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/misc/habanalabs/common/memory.c +1 -1
@@ -655,7 +655,7 @@ static u64 get_va_block(struct hl_device *hdev,continue ;/* -* In case hint address is 0, and arc_hints_range_reservation +* In case hint address is 0, and hints_range_reservation * property enabled, then avoid allocating va blocks from the * range reserved for hint addresses */
Signature · 1 line
2.25.1