habanalabs: fix MMU number of pages calculation
PATCH 1/4 habanalabs: fix MMU number of pages calculation
From: Omer Shpigelman <[email protected]>
The requested allocation size is 64bit, hence the number of requested pages and the total requested size should 64bit as well. This patch fixes all places where these are treated as 32bit.
--- drivers/misc/habanalabs/debugfs.c | 7 ++++--- drivers/misc/habanalabs/habanalabs.h | 8 ++++---- drivers/misc/habanalabs/memory.c | 29 ++++++++++++++-------------- 3 files changed, 23 insertions(+), 21 deletions(-)
drivers/misc/habanalabs/debugfs.c +4 -3
@@ -232,6 +232,7 @@ static int vm_show(struct seq_file *s, void *data)struct hl_vm_phys_pg_pack *phys_pg_pack =NULL ;enum vm_type_t *vm_type;bool once = true; +u64 j;int i;if (!dev_entry->hdev->mmu_enable)
@@ -260,7 +261,7 @@ static int vm_show(struct seq_file *s, void *data) }else { phys_pg_pack = hnode->ptr;seq_printf (s, -" 0x%-14llx %-10u %-4u\n" , +" 0x%-14llx %-10llu %-4u\n" , hnode->vaddr, phys_pg_pack->total_size, phys_pg_pack->handle); }
@@ -282,9 +283,9 @@ static int vm_show(struct seq_file *s, void *data) phys_pg_pack->page_size);seq_puts (s," physical address\n" );seq_puts (s,"---------------------\n" ); -for (i =0 ; i < phys_pg_pack->npages ; i++) { +for (j =0 ; j < phys_pg_pack->npages ; j++) {seq_printf (s," 0x%-14llx\n" , - phys_pg_pack->pages[i]); + phys_pg_pack->pages[j]); } }spin_unlock (&vm->idr_lock);
drivers/misc/habanalabs/habanalabs.h +4 -4
@@ -793,11 +793,11 @@ struct hl_vm_hash_node {* struct hl_vm_phys_pg_pack - physical page pack. * @vm_type: describes the type of the virtual area descriptor. * @pages: the physical page array. +* @npages: num physical pages in the pack. +* @total_size: total size of all the pages in this list. * @mapping_cnt: number of shared mappings. * @asid: the context related to this list. -* @npages: num physical pages in the pack. * @page_size: size of each page in the pack. -* @total_size: total size of all the pages in this list. * @flags: HL_MEM_* flags related to this list. * @handle: the provided handle related to this list. * @offset: offset from the first page.
@@ -807,11 +807,11 @@ struct hl_vm_hash_node {struct hl_vm_phys_pg_pack {enum vm_type_t vm_type;/* must be first */ u64 *pages; +u64 npages; +u64 total_size;atomic_t mapping_cnt;u32 asid; -u32 npages;u32 page_size; -u32 total_size;u32 flags;u32 handle;u32 offset;
drivers/misc/habanalabs/memory.c +15 -14
@@ -56,9 +56,9 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,struct hl_device *hdev = ctx->hdev;struct hl_vm *vm = &hdev->vm;struct hl_vm_phys_pg_pack *phys_pg_pack; -u64 paddr =0 ; -u32 total_size, num_pgs, num_curr_pgs, page_size, page_shift; -int handle, rc, i; +u64 paddr =0 , total_size, num_pgs, i; +u32 num_curr_pgs, page_size, page_shift; +int handle, rc;bool contiguous; num_curr_pgs =0 ;
@@ -73,7 +73,7 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, paddr = (u64 )gen_pool_alloc (vm->dram_pg_pool, total_size);if (!paddr) {dev_err (hdev->dev, -"failed to allocate %u huge contiguous pages\n" , +"failed to allocate %llu huge contiguous pages\n" , num_pgs);return -ENOMEM ; }
@@ -267,7 +267,7 @@ static void free_phys_pg_pack(struct hl_device *hdev, struct hl_vm_phys_pg_pack *phys_pg_pack) {struct hl_vm *vm = &hdev->vm; -int i; +u64 i;if (!phys_pg_pack->created_from_userptr) {if (phys_pg_pack->contiguous) {
@@ -519,7 +519,7 @@ static inline int add_va_block(struct hl_device *hdev,* - Return the start address of the virtual block */ static u64 get_va_block (struct hl_device *hdev, -struct hl_va_range *va_range,u32 size,u64 hint_addr, +struct hl_va_range *va_range,u64 size,u64 hint_addr,bool is_userptr) {struct hl_vm_va_block *va_block, *new_va_block =NULL ;
@@ -577,7 +577,8 @@ static u64 get_va_block(struct hl_device *hdev, }if (!new_va_block) { -dev_err (hdev->dev,"no available va block for size %u\n" , size); +dev_err (hdev->dev,"no available va block for size %llu\n" , + size); goto out; }
@@ -648,8 +649,8 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,struct hl_vm_phys_pg_pack *phys_pg_pack;struct scatterlist *sg;dma_addr_t dma_addr; -u64 page_mask; -u32 npages, total_npages, page_size =PAGE_SIZE ; +u64 page_mask, total_npages; +u32 npages, page_size =PAGE_SIZE ;bool first = true, is_huge_page_opt = true;int rc, i, j;
@@ -750,9 +751,9 @@ static int map_phys_page_pack(struct hl_ctx *ctx, u64 vaddr, struct hl_vm_phys_pg_pack *phys_pg_pack) {struct hl_device *hdev = ctx->hdev; -u64 next_vaddr = vaddr, paddr; +u64 next_vaddr = vaddr, paddr, mapped_pg_cnt =0 , i;u32 page_size = phys_pg_pack->page_size; -int i, rc =0 , mapped_pg_cnt =0 ; +int rc =0 ;for (i =0 ; i < phys_pg_pack->npages ; i++) { paddr = phys_pg_pack->pages[i];
@@ -764,7 +765,7 @@ static int map_phys_page_pack(struct hl_ctx *ctx, u64 vaddr, rc =hl_mmu_map (ctx, next_vaddr, paddr, page_size);if (rc) {dev_err (hdev->dev, -"map failed for handle %u, npages: %d, mapped: %d" , +"map failed for handle %u, npages: %llu, mapped: %llu" , phys_pg_pack->handle, phys_pg_pack->npages, mapped_pg_cnt); goto err;
@@ -985,10 +986,10 @@ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr)struct hl_vm_hash_node *hnode =NULL ;struct hl_userptr *userptr =NULL ;enum vm_type_t *vm_type; -u64 next_vaddr; +u64 next_vaddr, i;u32 page_size;bool is_userptr; -int i, rc; +int rc;/* protect from double entrance */ mutex_lock (&ctx->mem_hash_lock);
Signature · 1 line
2.17.1
PATCH 2/4 habanalabs: fix bug when mapping very large memory area
From: Omer Shpigelman <[email protected]>
This patch fixes a bug of allocating a too big memory size with kmalloc, which causes a failure.
In case of mapping a large memory block, an array of the relevant physical page addresses is allocated. If there are many pages the array might be too big to allocate with kmalloc, hence changing to kvmalloc.
--- drivers/misc/habanalabs/memory.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/misc/habanalabs/memory.c +5 -4
@@ -93,7 +93,7 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, phys_pg_pack->flags = args->flags; phys_pg_pack->contiguous = contiguous; - phys_pg_pack->pages =kcalloc (num_pgs,sizeof (u64),GFP_KERNEL ); + phys_pg_pack->pages =kvmalloc_array (num_pgs,sizeof (u64),GFP_KERNEL );if (!phys_pg_pack->pages) { rc = -ENOMEM ; goto pages_arr_err;
@@ -148,7 +148,7 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,gen_pool_free (vm->dram_pg_pool, phys_pg_pack->pages[i], page_size); -kfree (phys_pg_pack->pages); +kvfree (phys_pg_pack->pages); pages_arr_err:kfree (phys_pg_pack); pages_pack_err:
@@ -288,7 +288,7 @@ static void free_phys_pg_pack(struct hl_device *hdev, } } -kfree (phys_pg_pack->pages); +kvfree (phys_pg_pack->pages);kfree (phys_pg_pack); }
@@ -692,7 +692,8 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, page_mask = ~(((u64 ) page_size) -1 ); - phys_pg_pack->pages =kcalloc (total_npages,sizeof (u64),GFP_KERNEL ); + phys_pg_pack->pages =kvmalloc_array (total_npages,sizeof (u64), +GFP_KERNEL );if (!phys_pg_pack->pages) { rc = -ENOMEM ; goto page_pack_arr_mem_err;
Signature · 1 line
2.17.1
PATCH 3/4 habanalabs: complete user context cleanup before hard reset
In reply to Oded GabbayFrom: Omer Shpigelman <[email protected]>
This patch fixes a bug which led to a crash during hard reset flow. Before a hard reset is executed, we wait a few seconds for the user context cleanup to complete.
If it wasn't completed, we kill the user process and move on to the reset flow.
Upon killing the user process, the context cleanup flow begins and may take a while due to MMU unmaps.
Meanwhile, in the driver reset flow, we change the PCI DRAM bar location which can interfere with the MMU that uses the bar.
If the context cleanup flow didn't finish quickly, a crash may occur due to PCI DRAM bar mislocation during the MMU unmap.
Hence adding a wait between killing the user process and the start of the reset flow.
--- drivers/misc/habanalabs/device.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-)
drivers/misc/habanalabs/device.c +23 -1
@@ -11,6 +11,8 @@#include <linux/sched/signal.h> #include <linux/hwmon.h> +#define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10) +bool hl_device_disabled_or_in_reset (struct hl_device *hdev) {if ((hdev->disabled) || (atomic_read (&hdev->in_reset)))
@@ -462,9 +464,16 @@ static void hl_device_hard_reset_pending(struct work_struct *work)struct hl_device_reset_work *device_reset_work = container_of(work, struct hl_device_reset_work, reset_work);struct hl_device *hdev = device_reset_work->hdev; -u16 pending_cnt =HL_PENDING_RESET_PER_SEC ; +u16 pending_total, pending_cnt;struct task_struct *task =NULL ; +if (hdev->pldm) + pending_total =HL_PLDM_PENDING_RESET_PER_SEC ; +else + pending_total =HL_PENDING_RESET_PER_SEC ; + + pending_cnt = pending_total; +/* Flush all processes that are inside hl_open */ mutex_lock (&hdev->fd_open_cnt_lock);
@@ -489,6 +498,19 @@ static void hl_device_hard_reset_pending(struct work_struct *work) } } + pending_cnt = pending_total; + +while ((atomic_read (&hdev->fd_open_cnt)) && (pending_cnt)) { + + pending_cnt--; + +ssleep (1 ); + } + +if (atomic_read (&hdev->fd_open_cnt)) +dev_crit (hdev->dev, +"Going to hard reset with open user contexts\n" ); +mutex_unlock (&hdev->fd_open_cnt_lock);hl_device_reset (hdev, true, true);
Signature · 1 line
2.17.1
PATCH 4/4 habanalabs: fix mapping with page size bigger than 4KB
In reply to Oded GabbayFrom: Omer Shpigelman <[email protected]>
This patch fixes the mapping of virtual address to physical addresses on architectures where PAGE_SIZE is bigger than 4KB.
The break down to the device page size was done only for the virtual address while it should have been done for the physical address as well. As a result virtual addresses were mapped to wrong physical address. The fix is to apply the break down for the physical addresses as well in order to get correct mappings.
--- drivers/misc/habanalabs/mmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/misc/habanalabs/mmu.c +4 -2
@@ -832,7 +832,7 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,int hl_mmu_map (struct hl_ctx *ctx,u64 virt_addr,u64 phys_addr,u32 page_size) {struct hl_device *hdev = ctx->hdev; -u64 real_virt_addr; +u64 real_virt_addr, real_phys_addr;u32 real_page_size, npages;int i, rc, mapped_cnt =0 ;
@@ -857,14 +857,16 @@ int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size) npages = page_size / real_page_size; real_virt_addr = virt_addr; + real_phys_addr = phys_addr;for (i =0 ; i < npages ; i++) { - rc =_hl_mmu_map (ctx, real_virt_addr, phys_addr, + rc =_hl_mmu_map (ctx, real_virt_addr, real_phys_addr, real_page_size);if (rc) goto err; real_virt_addr += real_page_size; + real_phys_addr += real_page_size; mapped_cnt++; }
Signature · 1 line
2.17.1