mm/vmalloc: fix vmalloc_dump_obj VA lookup

From: Ye Liu <[email protected]>

vmalloc_dump_obj() uses PAGE_ALIGN() to normalize the input address before looking it up in the per-node busy tree. PAGE_ALIGN() rounds up, which can push an address in the last page of a vmalloc allocation to va_end -- outside the [va_start, va_end) range that __find_vmap_area() searches. This causes the lookup to miss the VA and return false, degrading diagnostic output in OOM dumps and KASAN reports to the less informative "vmalloc memory" fallback.

The upward alignment can also change the addr_to_node() mapping when the page boundary crosses a vmap zone boundary, causing the search to hit the wrong node entirely.

Use PAGE_ALIGN_DOWN() instead, which rounds down to the page containing the address. This keeps the address within the VA range and preserves the correct node mapping.

Signed-off-by Ye Liu <[email protected]>

---
 mm/vmalloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

mm/vmalloc.c +1 -1

index 859e6d2d57a3..df42d8a6f058 100644
@@ -5277,7 +5277,7 @@ bool vmalloc_dump_obj(void *object)
 	unsigned long addr;
 	unsigned long nr_pages;
 
-	addr = PAGE_ALIGN((unsigned long) object);
+	addr = PAGE_ALIGN_DOWN((unsigned long) object);
 	vn = addr_to_node(addr);
 
 	if (!spin_trylock(&vn->busy.lock))

Signature · 2 lines

-- 
2.25.1

From: Ye Liu <[email protected]>

vmalloc_dump_obj() searches only one vmap node (addr_to_node(addr)), but a vmalloc allocation may span multiple vmap zones. The VA is stored in only one node's rb-tree (addr_to_node(va_start)), so an object pointer in a different zone than va_start maps to a different node and the search misses. This affects any allocation larger than vmap_zone_size (64 KiB) on multi-CPU systems.

Iterate all vmap nodes using for_each_vmap_node, like find_vmap_area() does, but with spin_trylock instead of spin_lock as this function can be called from atomic dump contexts (OOM, KASAN, RCU).

Signed-off-by Ye Liu <[email protected]>

---
 mm/vmalloc.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

mm/vmalloc.c +18 -6

index df42d8a6f058..30c610f678dc 100644
@@ -5278,17 +5278,29 @@ bool vmalloc_dump_obj(void *object)
 	unsigned long nr_pages;
 
 	addr = PAGE_ALIGN_DOWN((unsigned long) object);
-	vn = addr_to_node(addr);
 
-	if (!spin_trylock(&vn->busy.lock))
-		return false;
+	/*
+	 * A vmalloc allocation may span multiple vmap zones, so the
+	 * node whose rb-tree holds the VA may differ from the node
+	 * the address maps to.  Search all nodes.  Use trylock as
+	 * this function can be called from atomic dump contexts.
+	 */
+	va = NULL;
+	for_each_vmap_node(vn) {
+		if (!spin_trylock(&vn->busy.lock))
+			continue;
+
+		va = __find_vmap_area(addr, &vn->busy.root);
+		if (va && va->vm)
+			break;
 
-	va = __find_vmap_area(addr, &vn->busy.root);
-	if (!va || !va->vm) {
 		spin_unlock(&vn->busy.lock);
-		return false;
+		va = NULL;
 	}
 
+	if (!va)
+		return false;
+
 	vm = va->vm;
 	addr = (unsigned long) vm->addr;
 	caller = vm->caller;

Signature · 2 lines

-- 
2.25.1

Quoted 62 lines · Ye Liu

From: Ye Liu <[email protected]>

vmalloc_dump_obj() searches only one vmap node (addr_to_node(addr)), but a vmalloc allocation may span multiple vmap zones. The VA is stored in only one node's rb-tree (addr_to_node(va_start)), so an object pointer in a different zone than va_start maps to a different node and the search misses. This affects any allocation larger than vmap_zone_size (64 KiB) on multi-CPU systems.

Iterate all vmap nodes using for_each_vmap_node, like find_vmap_area() does, but with spin_trylock instead of spin_lock as this function can be called from atomic dump contexts (OOM, KASAN, RCU).

Signed-off-by Ye Liu <[email protected]>

---
 mm/vmalloc.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

mm/vmalloc.c +18 -6

index df42d8a6f058..30c610f678dc 100644
@@ -5278,17 +5278,29 @@ bool vmalloc_dump_obj(void *object)
 	unsigned long nr_pages;
 
 	addr = PAGE_ALIGN_DOWN((unsigned long) object);
-	vn = addr_to_node(addr);
 
-	if (!spin_trylock(&vn->busy.lock))
-		return false;
+	/*
+	 * A vmalloc allocation may span multiple vmap zones, so the
+	 * node whose rb-tree holds the VA may differ from the node
+	 * the address maps to.  Search all nodes.  Use trylock as
+	 * this function can be called from atomic dump contexts.
+	 */
+	va = NULL;
+	for_each_vmap_node(vn) {
+		if (!spin_trylock(&vn->busy.lock))
+			continue;
+
+		va = __find_vmap_area(addr, &vn->busy.root);
+		if (va && va->vm)
+			break;
 
-	va = __find_vmap_area(addr, &vn->busy.root);
-	if (!va || !va->vm) {
 		spin_unlock(&vn->busy.lock);
-		return false;
+		va = NULL;
 	}
 
+	if (!va)
+		return false;
+
 	vm = va->vm;
 	addr = (unsigned long) vm->addr;
 	caller = vm->caller;

Signature · 2 lines

-- 
2.25.1

mm/vmalloc.c +44 -5

index 89c327a6ce7d..3719dc02dcaf 100644
@@ -2511,7 +2511,22 @@ static void free_unmap_vmap_area(struct vmap_area *va)
 	free_vmap_area_noflush(va);
 }
 
-struct vmap_area *find_vmap_area(unsigned long addr)
+static inline int next_vmap_node_id(int i)
+{
+	return (i + nr_vmap_nodes - 1) % nr_vmap_nodes;
+}
+
+enum vmap_lock_mode {
+	VMAP_LOCK,
+	VMAP_TRYLOCK,
+};
+
+/*
+ * Add a comment here.
+ */
+static struct vmap_area *
+find_vmap_area_lock(unsigned long addr, struct vmap_node **locked_vn,
+		enum vmap_lock_mode mode)
 {
 	struct vmap_node *vn;
 	struct vmap_area *va;

@@ -2534,16 +2549,40 @@ struct vmap_area *find_vmap_area(unsigned long addr)
 	 * addr is within 2 or 0 nodes we should do extra work.
 	 */
 	i = j = addr_to_node_id(addr);
+
 	do {
 		vn = &vmap_nodes[i];
 
-		spin_lock(&vn->busy.lock);
+		if (mode == VMAP_LOCK) {
+			spin_lock(&vn->busy.lock);
+		} else {
+			if (!spin_trylock(&vn->busy.lock))
+				continue;
+		}
+
 		va = __find_vmap_area(addr, &vn->busy.root);
+		if (va) {
+			*locked_vn = vn;
+			return va;
+		}
+
 		spin_unlock(&vn->busy.lock);
+	} while ((i = next_vmap_node_id(i)) != j);
 
-		if (va)
-			return va;
-	} while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j);
+	*locked_vn = NULL;
+	return NULL;
+}
+
+struct vmap_area *find_vmap_area(unsigned long addr)
+{
+	struct vmap_node *vn;
+	struct vmap_area *va;
+
+	va = find_vmap_area_lock(addr, &vn, VMAP_LOCK);
+	if (va) {
+		spin_unlock(&vn->busy.lock);
+		return va;
+	}
 
 	return NULL;
 }

and we use the helper in the vmalloc_dump_obj()?

--
Uladzislau Rezki

From: Ye Liu <[email protected]>

vmalloc_dump_obj() unconditionally searches all vmap nodes even when called with a non-vmalloc address (e.g. a slab or stack pointer from mem_dump_obj()). Add an is_vmalloc_or_module_addr() check at the entry to avoid the unnecessary per-node trylock and rb-tree traversal.

Use is_vmalloc_or_module_addr() rather than is_vmalloc_addr() because module, BPF, and execmem allocations reside in MODULES_VADDR..MODULES_END on x86_64, arm64, and riscv -- outside VMALLOC_START..VMALLOC_END -- but are still tracked in the same vmap_nodes rb-tree.

Signed-off-by Ye Liu <[email protected]>

---
 mm/vmalloc.c | 3 +++
 1 file changed, 3 insertions(+)

mm/vmalloc.c +3 -0

index 30c610f678dc..d8095b558365 100644
@@ -5277,6 +5277,9 @@ bool vmalloc_dump_obj(void *object)
 	unsigned long addr;
 	unsigned long nr_pages;
 
+	if (!is_vmalloc_or_module_addr(object))
+		return false;
+
 	addr = PAGE_ALIGN_DOWN((unsigned long) object);
 
 	/*

Signature · 2 lines

-- 
2.25.1

Quoted 35 lines · Ye Liu

From: Ye Liu <[email protected]>

vmalloc_dump_obj() unconditionally searches all vmap nodes even when called with a non-vmalloc address (e.g. a slab or stack pointer from mem_dump_obj()). Add an is_vmalloc_or_module_addr() check at the entry to avoid the unnecessary per-node trylock and rb-tree traversal.

Use is_vmalloc_or_module_addr() rather than is_vmalloc_addr() because module, BPF, and execmem allocations reside in MODULES_VADDR..MODULES_END on x86_64, arm64, and riscv -- outside VMALLOC_START..VMALLOC_END -- but are still tracked in the same vmap_nodes rb-tree.

Signed-off-by Ye Liu <[email protected]>

---
 mm/vmalloc.c | 3 +++
 1 file changed, 3 insertions(+)

mm/vmalloc.c +3 -0

index 30c610f678dc..d8095b558365 100644
@@ -5277,6 +5277,9 @@ bool vmalloc_dump_obj(void *object)
 	unsigned long addr;
 	unsigned long nr_pages;
 
+	if (!is_vmalloc_or_module_addr(object))
+		return false;
+
 	addr = PAGE_ALIGN_DOWN((unsigned long) object);
 
 	/*

Signature · 2 lines

-- 
2.25.1

Do we need this check? If it is not the vmalloc address, we just return noting. Another question is why do you want is_vmalloc_or_module_addr()?

vmalloc_dump_obj() is about VMALLOC_START..VMALLOC_END, IMO.

There are only two users of it and both rely on the VMALLOC_START..VMALLOC_END range:

<snip>
*** mm/kasan/report.c:
print_address_description[403] if (!vmalloc_dump_obj(addr))

*** mm/util.c:
mem_dump_obj[1096] if (vmalloc_dump_obj(object))
<snip>

	if (is_vmalloc_addr(addr)) {
		pr_err("The buggy address belongs to a");
		if (!vmalloc_dump_obj(addr))
			pr_cont(" vmalloc virtual mapping\n");
		page = vmalloc_to_page(addr);
	}

and

<snip>

	if (vmalloc_dump_obj(object))
		return;

	if (is_vmalloc_addr(object))
		type = "vmalloc memory";

<snip>

--
Uladzislau Rezki