habanalabs: improve hard reset procedure
PATCH 1/1 habanalabs: improve hard reset procedure
From: Ofir Bitton <[email protected]>
We want to handle the scenario in which the driver was not able to kill all user processes due to many memory mappings. We need to retry again after some period while releasing the cores. The devices will be unusable and "in-reset" status during that time.
--- drivers/misc/habanalabs/common/device.c | 128 ++++++++++++++------ drivers/misc/habanalabs/common/habanalabs.h | 16 ++- 2 files changed, 106 insertions(+), 38 deletions(-)
drivers/misc/habanalabs/common/device.c +92 -36
@@ -13,8 +13,6 @@#include <linux/hwmon.h> #include <uapi/misc/habanalabs.h> -#define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10) -enum hl_device_status hl_device_status (struct hl_device *hdev) {enum hl_device_status status;
@@ -256,6 +254,26 @@ static void device_cdev_sysfs_del(struct hl_device *hdev)cdev_device_del (&hdev->cdev, hdev->dev); } +static void device_hard_reset_pending (struct work_struct *work) +{ +struct hl_device_reset_work *device_reset_work = + container_of(work, struct hl_device_reset_work, + reset_work.work); +struct hl_device *hdev = device_reset_work->hdev; +int rc; + + rc =hl_device_reset (hdev, true, true); +if ((rc == -EBUSY ) && !hdev->device_fini_pending) { +dev_info (hdev->dev, +"Could not reset device. will try again in %u seconds" , +HL_PENDING_RESET_PER_SEC ); + +queue_delayed_work (device_reset_work->wq, + &device_reset_work->reset_work, +msecs_to_jiffies (HL_PENDING_RESET_PER_SEC *1000 )); + } +} +/* * device_early_init - do some early initialization for the habanalabs device *
@@ -340,6 +358,19 @@ static int device_early_init(struct hl_device *hdev)hl_cb_mgr_init (&hdev->kernel_cb_mgr); + hdev->device_reset_work.wq = +create_singlethread_workqueue ("hl_device_reset" ); +if (!hdev->device_reset_work.wq) { + rc = -ENOMEM ; +dev_err (hdev->dev,"Failed to create device reset WQ\n" ); + goto free_cb_mgr; + } + +INIT_DELAYED_WORK (&hdev->device_reset_work.reset_work, + device_hard_reset_pending); + hdev->device_reset_work.hdev = hdev; + hdev->device_fini_pending =0 ; +mutex_init (&hdev->send_cpu_message_lock);mutex_init (&hdev->debug_lock);mutex_init (&hdev->mmu_cache_lock);
@@ -351,6 +382,8 @@ static int device_early_init(struct hl_device *hdev)return 0 ; +free_cb_mgr: +hl_cb_mgr_fini (hdev, &hdev->kernel_cb_mgr); free_idle_busy_ts_arr:kfree (hdev->idle_busy_ts_arr); free_chip_info:
@@ -393,6 +426,7 @@ static void device_early_fini(struct hl_device *hdev)kfree (hdev->hl_chip_info);destroy_workqueue (hdev->eq_wq); +destroy_workqueue (hdev->device_reset_work.wq);for (i =0 ; i < hdev->asic_prop.completion_queues_count ; i++)destroy_workqueue (hdev->cq_wq[i]);
@@ -771,16 +805,12 @@ int hl_device_resume(struct hl_device *hdev)return rc; } -static int device_kill_open_processes (struct hl_device *hdev) +static int device_kill_open_processes (struct hl_device *hdev,u32 timeout) { -u16 pending_total, pending_cnt;struct hl_fpriv *hpriv;struct task_struct *task =NULL ; +u32 pending_cnt; -if (hdev->pldm) - pending_total =HL_PLDM_PENDING_RESET_PER_SEC ; -else - pending_total =HL_PENDING_RESET_PER_SEC ;/* Giving time for user to close FD, and for processes that are inside * hl_device_open to finish
@@ -788,6 +818,19 @@ static int device_kill_open_processes(struct hl_device *hdev)if (!list_empty (&hdev->fpriv_list))ssleep (1 ); +if (timeout) { + pending_cnt = timeout; + }else { +if (hdev->process_kill_trial_cnt) { +/* Processes have been already killed */ + pending_cnt =1 ; + goto wait_for_processes; + }else { +/* Wait a small period after process kill */ + pending_cnt =HL_PENDING_RESET_PER_SEC ; + } + } +mutex_lock (&hdev->fpriv_list_lock);/* This section must be protected because we are dereferencing
@@ -816,29 +859,27 @@ static int device_kill_open_processes(struct hl_device *hdev)* continuing with the reset. */ - pending_cnt = pending_total; - +wait_for_processes:while ((!list_empty (&hdev->fpriv_list)) && (pending_cnt)) { -dev_info (hdev->dev, -"Waiting for all user contexts to get closed before hard reset\n" ); +dev_dbg (hdev->dev, +"Waiting for all unmap operations to finish before hard reset\n" ); pending_cnt--;ssleep (1 ); } -return list_empty (&hdev->fpriv_list) ?0 : -EBUSY ; -} +/* All processes exited successfully */ +if (list_empty (&hdev->fpriv_list)) +return 0 ; -static void device_hard_reset_pending (struct work_struct *work) -{ -struct hl_device_reset_work *device_reset_work = - container_of(work, struct hl_device_reset_work, reset_work); -struct hl_device *hdev = device_reset_work->hdev; +/* Give up waiting for processes to exit */ +if (hdev->process_kill_trial_cnt ==HL_PENDING_RESET_MAX_TRIALS ) +return -ETIME ; -hl_device_reset (hdev, true, true); + hdev->process_kill_trial_cnt++; -kfree (device_reset_work); +return -EBUSY ; }/*
@@ -875,6 +916,10 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset, hard_reset = true; } +/* Re-entry of reset thread */ +if (from_hard_reset_thread && hdev->process_kill_trial_cnt) + goto kill_processes; +/* * Prevent concurrency in this function - only one reset should be * done at any given time. Only need to perform this if we didn't
@@ -920,26 +965,17 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset, again:if ((hard_reset) && (!from_hard_reset_thread)) { -struct hl_device_reset_work *device_reset_work; - hdev->hard_reset_pending = true; - device_reset_work =kzalloc (sizeof (*device_reset_work), -GFP_ATOMIC ); -if (!device_reset_work) { - rc = -ENOMEM ; - goto out_err; - } + hdev->process_kill_trial_cnt =0 ;/* * Because the reset function can't run from interrupt or * from heartbeat work, we need to call the reset function * from a dedicated work */ -INIT_WORK (&device_reset_work->reset_work, - device_hard_reset_pending); - device_reset_work->hdev = hdev; -schedule_work (&device_reset_work->reset_work); +queue_delayed_work (hdev->device_reset_work.wq, + &hdev->device_reset_work.reset_work,0 );return 0 ; }
@@ -965,12 +1001,25 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,/* Go over all the queues, release all CS and their jobs */ hl_cs_rollback_all (hdev); +kill_processes:if (hard_reset) {/* Kill processes here after CS rollback. This is because the * process can't really exit until all its CSs are done, which * is what we do in cs rollback */ - rc =device_kill_open_processes (hdev); + rc =device_kill_open_processes (hdev,0 ); + +if (rc == -EBUSY ) { +if (hdev->device_fini_pending) { +dev_crit (hdev->dev, +"Failed to kill all open processes, stopping hard reset\n" ); + goto out_err; + } + +/* signal reset thread to reschedule */ +return rc; + } +if (rc) {dev_crit (hdev->dev,"Failed to kill all open processes, stopping hard reset\n" );
@@ -1408,11 +1457,14 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)*/ void hl_device_fini (struct hl_device *hdev) { -int i, rc;ktime_t timeout; +int i, rc;dev_info (hdev->dev,"Removing device\n" ); + hdev->device_fini_pending =1 ; +flush_delayed_work (&hdev->device_reset_work.reset_work); +/* * This function is competing with the reset function, so try to * take the reset atomic and if we are already in middle of reset,
@@ -1468,7 +1520,11 @@ void hl_device_fini(struct hl_device *hdev)* can't really exit until all its CSs are done, which is what we * do in cs rollback */ - rc =device_kill_open_processes (hdev); +dev_info (hdev->dev, +"Waiting for all processes to exit (timeout of %u seconds)" , +HL_PENDING_RESET_LONG_SEC ); + + rc =device_kill_open_processes (hdev,HL_PENDING_RESET_LONG_SEC );if (rc)dev_crit (hdev->dev,"Failed to kill all open processes\n" );
drivers/misc/habanalabs/common/habanalabs.h +14 -2
@@ -40,7 +40,9 @@#define HL_MMAP_OFFSET_VALUE_MASK (0x3FFFFFFFFFFFull >> PAGE_SHIFT)#define HL_MMAP_OFFSET_VALUE_GET (off) (off & HL_MMAP_OFFSET_VALUE_MASK) -#define HL_PENDING_RESET_PER_SEC 30 +#define HL_PENDING_RESET_PER_SEC 10 +#define HL_PENDING_RESET_MAX_TRIALS 60/* 10 minutes */ +#define HL_PENDING_RESET_LONG_SEC 60#define HL_HARD_RESET_MAX_TIMEOUT 120
@@ -1588,11 +1590,13 @@ struct hwmon_chip_info;/** * struct hl_device_reset_work - reset workqueue task wrapper. +* @wq: work queue for device reset procedure. * @reset_work: reset work to be done. * @hdev: habanalabs device structure. */ struct hl_device_reset_work { -struct work_struct reset_work; +struct workqueue_struct *wq; +struct delayed_work reset_work;struct hl_device *hdev; };
@@ -1691,6 +1695,7 @@ struct hl_mmu_funcs {* @dev_ctrl: related kernel device structure for the control device * @work_freq: delayed work to lower device frequency if possible. * @work_heartbeat: delayed work for CPU-CP is-alive check. +* @device_reset_work: delayed work which performs hard reset * @asic_name: ASIC specific name. * @asic_type: ASIC specific type. * @completion_queue: array of hl_cq.
@@ -1790,6 +1795,10 @@ struct hl_mmu_funcs {* @supports_cb_mapping: is mapping a CB to the device's MMU supported. * @needs_reset: true if reset_on_lockup is false and device should be reset * due to lockup. +* @process_kill_trial_cnt: number of trials reset thread tried killing +* user processes +* @device_fini_pending: true if device_fini was called and might be +* waiting for the reset thread to finish */ struct hl_device {struct pci_dev *pdev;
@@ -1802,6 +1811,7 @@ struct hl_device {struct device *dev_ctrl;struct delayed_work work_freq;struct delayed_work work_heartbeat; +struct hl_device_reset_work device_reset_work;char asic_name[HL_STR_MAX ];char status[HL_DEV_STS_MAX ][HL_STR_MAX ];enum hl_asic_type asic_type;
@@ -1894,6 +1904,8 @@ struct hl_device {u8 supports_soft_reset;u8 supports_cb_mapping;u8 needs_reset; +u8 process_kill_trial_cnt; +u8 device_fini_pending;/* Parameters for bring-up */ u64 nic_ports_mask;
Signature · 1 line
2.17.1
PATCH 1/1 habanalabs: refactor mmu va_range db structure
From: Ofir Bitton <[email protected]>
Use an array of va_ranges instead of keeping each va_range separately, we do this for better readability and in order to support access to a specific range in a much elegant manner.
--- drivers/misc/habanalabs/common/habanalabs.h | 24 ++-- drivers/misc/habanalabs/common/memory.c | 130 ++++++++++---------- 2 files changed, 82 insertions(+), 72 deletions(-)
drivers/misc/habanalabs/common/habanalabs.h +17 -7
@@ -953,18 +953,33 @@ struct hl_asic_funcs {#define HL_KERNEL_ASID_ID 0 +/** +* enum hl_va_range_type - virtual address range type. +* @HL_VA_RANGE_TYPE_HOST: range type of host pages +* @HL_VA_RANGE_TYPE_HOST_HUGE: range type of host huge pages +* @HL_VA_RANGE_TYPE_DRAM: range type of dram pages +*/ +enum hl_va_range_type { +HL_VA_RANGE_TYPE_HOST , +HL_VA_RANGE_TYPE_HOST_HUGE , +HL_VA_RANGE_TYPE_DRAM , +HL_VA_RANGE_TYPE_MAX +}; +/** * struct hl_va_range - virtual addresses range. * @lock: protects the virtual addresses list. * @list: list of virtual addresses blocks available for mappings. * @start_addr: range start address. * @end_addr: range end address. +* @page_size: page size of this va range. */ struct hl_va_range {struct mutex lock;struct list_head list;u64 start_addr;u64 end_addr; +u32 page_size; };/**
@@ -993,10 +1008,7 @@ struct hl_cs_counters_atomic {* @refcount: reference counter for the context. Context is released only when * this hits 0l. It is incremented on CS and CS_WAIT. * @cs_pending: array of hl fence objects representing pending CS. -* @host_va_range: holds available virtual addresses for host mappings. -* @host_huge_va_range: holds available virtual addresses for host mappings -* with huge pages. -* @dram_va_range: holds available virtual addresses for DRAM mappings. +* @va_range: holds available virtual addresses for host and dram mappings. * @mem_hash_lock: protects the mem_hash. * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the * MMU hash or walking the PGT requires talking this lock.
@@ -1028,9 +1040,7 @@ struct hl_ctx {struct hl_device *hdev;struct kref refcount;struct hl_fence **cs_pending; -struct hl_va_range *host_va_range; -struct hl_va_range *host_huge_va_range; -struct hl_va_range *dram_va_range; +struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX ];struct mutex mem_hash_lock;struct mutex mmu_lock;struct list_head debugfs_list;
drivers/misc/habanalabs/common/memory.c +65 -65
250 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 1/1 habanalabs: Rename hw_queues_mirror to cs_mirror
In reply to Oded GabbayFrom: Tomer Tayar <[email protected]>
Future command submission types might be submitted to HW not via the QMAN queues path. However, it would be still required to have the TDR mechanism for these CS, and thus the patch renames the TDR fields and replaces the hw_queues_ prefix with cs_.
--- .../habanalabs/common/command_submission.c | 18 ++++++++---------- drivers/misc/habanalabs/common/device.c | 4 ++-- drivers/misc/habanalabs/common/habanalabs.h | 8 ++++---- drivers/misc/habanalabs/common/hw_queue.c | 12 ++++++------ 4 files changed, 20 insertions(+), 22 deletions(-)
drivers/misc/habanalabs/common/command_submission.c +8 -10
@@ -343,25 +343,24 @@ static void cs_do_release(struct kref *ref)/* Need to update CI for internal queues */ hl_int_hw_queue_update_ci (cs); -spin_lock (&hdev->hw_queues_mirror_lock); -/* remove CS from hw_queues mirror list */ +/* remove CS from CS mirror list */ +spin_lock (&hdev->cs_mirror_lock);list_del_init (&cs->mirror_node); -spin_unlock (&hdev->hw_queues_mirror_lock); +spin_unlock (&hdev->cs_mirror_lock);/* Don't cancel TDR in case this CS was timedout because we might be * running from the TDR context */ -if (!cs->timedout && - hdev->timeout_jiffies !=MAX_SCHEDULE_TIMEOUT ) { +if (!cs->timedout && hdev->timeout_jiffies !=MAX_SCHEDULE_TIMEOUT ) {struct hl_cs *next;if (cs->tdr_active)cancel_delayed_work_sync (&cs->work_tdr); -spin_lock (&hdev->hw_queues_mirror_lock); +spin_lock (&hdev->cs_mirror_lock);/* queue TDR for next CS */ - next =list_first_entry_or_null (&hdev->hw_queues_mirror_list, + next =list_first_entry_or_null (&hdev->cs_mirror_list, struct hl_cs, mirror_node);if (next && !next->tdr_active) {
@@ -370,7 +369,7 @@ static void cs_do_release(struct kref *ref) hdev->timeout_jiffies); } -spin_unlock (&hdev->hw_queues_mirror_lock); +spin_unlock (&hdev->cs_mirror_lock); } out:
@@ -534,8 +533,7 @@ void hl_cs_rollback_all(struct hl_device *hdev)flush_workqueue (hdev->cq_wq[i]);/* Make sure we don't have leftovers in the H/W queues mirror list */ -list_for_each_entry_safe (cs, tmp, &hdev->hw_queues_mirror_list, - mirror_node) { +list_for_each_entry_safe (cs, tmp, &hdev->cs_mirror_list, mirror_node) {cs_get (cs); cs->aborted = true; dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n",
drivers/misc/habanalabs/common/device.c +2 -2
@@ -343,8 +343,8 @@ static int device_early_init(struct hl_device *hdev)mutex_init (&hdev->send_cpu_message_lock);mutex_init (&hdev->debug_lock);mutex_init (&hdev->mmu_cache_lock); -INIT_LIST_HEAD (&hdev->hw_queues_mirror_list); -spin_lock_init (&hdev->hw_queues_mirror_lock); +INIT_LIST_HEAD (&hdev->cs_mirror_list); +spin_lock_init (&hdev->cs_mirror_lock);INIT_LIST_HEAD (&hdev->fpriv_list);mutex_init (&hdev->fpriv_list_lock);atomic_set (&hdev->in_reset,0 );
drivers/misc/habanalabs/common/habanalabs.h +4 -4
@@ -1699,8 +1699,8 @@ struct hl_mmu_funcs {* @eq_wq: work queue of event queue for executing work in process context. * @kernel_ctx: Kernel driver context structure. * @kernel_queues: array of hl_hw_queue. -* @hw_queues_mirror_list: CS mirror list for TDR. -* @hw_queues_mirror_lock: protects hw_queues_mirror_list. +* @cs_mirror_list: CS mirror list for TDR. +* @cs_mirror_lock: protects cs_mirror_list. * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs. * @event_queue: event queue for IRQ from CPU-CP. * @dma_pool: DMA pool for small allocations.
@@ -1810,8 +1810,8 @@ struct hl_device {struct workqueue_struct *eq_wq;struct hl_ctx *kernel_ctx;struct hl_hw_queue *kernel_queues; -struct list_head hw_queues_mirror_list; -spinlock_t hw_queues_mirror_lock; +struct list_head cs_mirror_list; +spinlock_t cs_mirror_lock;struct hl_cb_mgr kernel_cb_mgr;struct hl_eq event_queue;struct dma_pool *dma_pool;
drivers/misc/habanalabs/common/hw_queue.c +6 -6
@@ -578,20 +578,20 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs)else if (cs->type ==CS_TYPE_COLLECTIVE_WAIT ) hdev->asic_funcs->collective_wait_init_cs (cs); -spin_lock (&hdev->hw_queues_mirror_lock); -list_add_tail (&cs->mirror_node, &hdev->hw_queues_mirror_list); +spin_lock (&hdev->cs_mirror_lock); +list_add_tail (&cs->mirror_node, &hdev->cs_mirror_list);/* Queue TDR if the CS is the first entry and if timeout is wanted */ if ((hdev->timeout_jiffies !=MAX_SCHEDULE_TIMEOUT ) && - (list_first_entry (&hdev->hw_queues_mirror_list, + (list_first_entry (&hdev->cs_mirror_list, struct hl_cs, mirror_node) == cs)) { cs->tdr_active = true;schedule_delayed_work (&cs->work_tdr, hdev->timeout_jiffies); -spin_unlock (&hdev->hw_queues_mirror_lock); - }else { -spin_unlock (&hdev->hw_queues_mirror_lock); + } +spin_unlock (&hdev->cs_mirror_lock); +if (!hdev->cs_active_cnt++) {struct hl_device_idle_busy_ts *ts;
Signature · 1 line
2.17.1