Skip to contentSkip to conversation

bpf: Add memory usage for arena and selftest

PATCH Applied Jiayuan Chen 4 participants 12 messages Jul 17 – Jul 19

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

v2 cover bpf: Add memory usage for arena and selftest

arena is the only map type whose map_mem_usage() still returns 0, so bpftool and fdinfo always show 0 memlock for it no matter how many pages it holds. This series adds the accounting, runs the arena selftests serially so they don't OOM a small CI VM, and adds a test for it.

v1 -> v2:
- split the scratch_page -> arena refactor into a prep patch (no functional change)
- use WRITE_ONCE() for nr_pages to pair with the lockless reader
- reword the "run serially" commit message
- selftest: alloc prog returns 0 so the failure check is reached

v1: https://lore.kernel.org/bpf/[email protected]/

Jiayuan Chen (4):
bpf: Pass arena instead of scratch_page to the pte callbacks
bpf: Add memory usage for arena
selftests/bpf: Run arena tests serially selftests/bpf: Add tests for memory usage for arena

 kernel/bpf/arena.c                            |  27 ++--
 .../selftests/bpf/prog_tests/arena_atomics.c  |   2 +-
 .../bpf/prog_tests/arena_direct_value.c       |   2 +-
 .../selftests/bpf/prog_tests/arena_htab.c     |   2 +-
 .../selftests/bpf/prog_tests/arena_list.c     |   2 +-
 .../bpf/prog_tests/arena_mem_usage.c          | 122 ++++++++++++++++++
 .../bpf/prog_tests/arena_spin_lock.c          |   2 +-
 .../bpf/prog_tests/arena_strsearch.c          |   2 +-
 .../selftests/bpf/prog_tests/libarena.c       |   2 +-
 .../selftests/bpf/prog_tests/libarena_asan.c  |   2 +-
 .../selftests/bpf/progs/arena_mem_usage.c     |  40 ++++++
 11 files changed, 186 insertions(+), 19 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_mem_usage.c
Signature · 1 line
2.43.0
#1 lore raw

PATCH v2 1/4 bpf: Pass arena instead of scratch_page to the pte callbacks

Replace the scratch_page field in the pte-callback data with the arena pointer; later patches use other arena fields from these callbacks. No functional change.

---
 kernel/bpf/arena.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
kernel/bpf/arena.c +8 -9
index 80b7b8a69446..8dbc24460890 100644
@@ -143,14 +143,14 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr)
 }
 
 struct apply_range_data {
+	struct bpf_arena *arena;
 	struct page **pages;
-	struct page *scratch_page;
 	int i;
 };
 
 struct clear_range_data {
+	struct bpf_arena *arena;
 	struct llist_head *free_pages;
-	struct page *scratch_page;
 };
 
 static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
@@ -180,7 +180,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
 
 		if (pte_none(old))
 			continue;
-		if (WARN_ON_ONCE(pte_page(old) != d->scratch_page))
+		if (WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page))
 			return -EBUSY;
 		ptep_get_and_clear(&init_mm, addr, pte);
 		flush_tlb_before_set(addr);
@@ -227,7 +227,7 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data)
 	 * scratches its PTE. A later bpf_arena_free_pages() over that range walks
 	 * here. Without the skip, scratch_page would be freed.
 	 */
-	if (page == d->scratch_page)
+	if (page == d->arena->scratch_page)
 		return 0;
 
 	__llist_add(&page->pcp_llist, d->free_pages);
@@ -506,8 +506,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
 	if (ret)
 		goto out_sigsegv_memcg;
 
-	struct apply_range_data data = { .pages = &page, .i = 0,
-					 .scratch_page = arena->scratch_page };
+	struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 };
 	/* Account into memcg of the process that created bpf_arena */
 	ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page);
 	if (ret) {
@@ -696,8 +695,8 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
 		bpf_map_memcg_exit(old_memcg, new_memcg);
 		return 0;
 	}
+	data.arena = arena;
 	data.pages = pages;
-	data.scratch_page = arena->scratch_page;
 
 	if (raw_res_spin_lock_irqsave(&arena->spinlock, flags))
 		goto out_free_pages;
@@ -873,8 +872,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 	range_tree_set(&arena->rt, pgoff, page_cnt);
 
 	init_llist_head(&free_pages);
+	cdata.arena = arena;
 	cdata.free_pages = &free_pages;
-	cdata.scratch_page = arena->scratch_page;
 	/* clear ptes and collect struct pages */
 	apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,
 				     apply_range_clear_cb, &cdata);
@@ -981,8 +980,8 @@ static void arena_free_worker(struct work_struct *work)
 	bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
 
 	init_llist_head(&free_pages);
+	cdata.arena = arena;
 	cdata.free_pages = &free_pages;
-	cdata.scratch_page = arena->scratch_page;
 	arena_vm_start = bpf_arena_get_kern_vm_start(arena);
 	user_vm_start = bpf_arena_get_user_vm_start(arena);
 
Signature · 1 line
2.43.0
reviewer Reviewed-by on v2 1/4
#2 lore raw

Re: [PATCH bpf-next v2 1/4] bpf: Pass arena instead of scratch_page to the pte callbacks

Quoted 5 lines · Jiayuan Chen

Replace the scratch_page field in the pte-callback data with the arena pointer; later patches use other arena fields from these callbacks. No functional change.

Quoted 83 lines
---
 kernel/bpf/arena.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
kernel/bpf/arena.c +8 -9
index 80b7b8a69446..8dbc24460890 100644
@@ -143,14 +143,14 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr)
 }
 
 struct apply_range_data {
+	struct bpf_arena *arena;
 	struct page **pages;
-	struct page *scratch_page;
 	int i;
 };
 
 struct clear_range_data {
+	struct bpf_arena *arena;
 	struct llist_head *free_pages;
-	struct page *scratch_page;
 };
 
 static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
@@ -180,7 +180,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
 
 		if (pte_none(old))
 			continue;
-		if (WARN_ON_ONCE(pte_page(old) != d->scratch_page))
+		if (WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page))
 			return -EBUSY;
 		ptep_get_and_clear(&init_mm, addr, pte);
 		flush_tlb_before_set(addr);
@@ -227,7 +227,7 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data)
 	 * scratches its PTE. A later bpf_arena_free_pages() over that range walks
 	 * here. Without the skip, scratch_page would be freed.
 	 */
-	if (page == d->scratch_page)
+	if (page == d->arena->scratch_page)
 		return 0;
 
 	__llist_add(&page->pcp_llist, d->free_pages);
@@ -506,8 +506,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
 	if (ret)
 		goto out_sigsegv_memcg;
 
-	struct apply_range_data data = { .pages = &page, .i = 0,
-					 .scratch_page = arena->scratch_page };
+	struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 };
 	/* Account into memcg of the process that created bpf_arena */
 	ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page);
 	if (ret) {
@@ -696,8 +695,8 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
 		bpf_map_memcg_exit(old_memcg, new_memcg);
 		return 0;
 	}
+	data.arena = arena;
 	data.pages = pages;
-	data.scratch_page = arena->scratch_page;
 
 	if (raw_res_spin_lock_irqsave(&arena->spinlock, flags))
 		goto out_free_pages;
@@ -873,8 +872,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 	range_tree_set(&arena->rt, pgoff, page_cnt);
 
 	init_llist_head(&free_pages);
+	cdata.arena = arena;
 	cdata.free_pages = &free_pages;
-	cdata.scratch_page = arena->scratch_page;
 	/* clear ptes and collect struct pages */
 	apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,
 				     apply_range_clear_cb, &cdata);
@@ -981,8 +980,8 @@ static void arena_free_worker(struct work_struct *work)
 	bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
 
 	init_llist_head(&free_pages);
+	cdata.arena = arena;
 	cdata.free_pages = &free_pages;
-	cdata.scratch_page = arena->scratch_page;
 	arena_vm_start = bpf_arena_get_kern_vm_start(arena);
 	user_vm_start = bpf_arena_get_user_vm_start(arena);
 
#3 lore raw

PATCH v2 2/4 bpf: Add memory usage for arena

In reply to Jiayuan Chen

arena is the only map type whose map_mem_usage() still returns 0, so "bpftool map show" and fdinfo always showed 0 memlock for an arena no matter how many pages it had.

Count the pages that are actually mapped into the arena: bump a counter in apply_range_set_cb() when a page goes in and drop it in apply_range_clear_cb() when a page goes out, both under the arena spinlock. map_mem_usage() then just returns nr_pages << PAGE_SHIFT.

Only real data pages are counted, not the scratch page.

---
To sashiko:
1. __bpf_alloc_page::can_alloc_pages already check whether we are under lock.
2. apply_to_page_range() does not allocate page tables here. arena_map_alloc() already did it.

---
 kernel/bpf/arena.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
kernel/bpf/arena.c +8 -2
index 8dbc24460890..f046e878f7ae 100644
@@ -55,8 +55,10 @@ struct bpf_arena {
 	struct vm_struct *kern_vm;
 	struct page *scratch_page;
 	struct range_tree rt;
-	/* protects rt */
+	/* protects rt and nr_pages */
 	rqspinlock_t spinlock;
+	/* number of pages currently populated in the arena */
+	u64 nr_pages;
 	struct list_head vma_list;
 	/* protects vma_list */
 	struct mutex lock;
@@ -196,6 +198,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
 	set_pte_at(&init_mm, addr, pte, pteval);
 #endif
 	d->i++;
+	WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1);
 	return 0;
 }
 
@@ -231,6 +234,7 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data)
 		return 0;
 
 	__llist_add(&page->pcp_llist, d->free_pages);
+	WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages - 1);
 	return 0;
 }
 
@@ -413,7 +417,9 @@ static int arena_map_check_btf(struct bpf_map *map, const struct btf *btf,
 
 static u64 arena_map_mem_usage(const struct bpf_map *map)
 {
-	return 0;
+	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
+
+	return (u64)READ_ONCE(arena->nr_pages) << PAGE_SHIFT;
 }
 
 struct vma_list {
Signature · 1 line
2.43.0
reviewer Reviewed-by on v2 2/4
#4 lore raw

Re: [PATCH bpf-next v2 2/4] bpf: Add memory usage for arena

Quoted 4 lines · Jiayuan Chen

arena is the only map type whose map_mem_usage() still returns 0, so "bpftool map show" and fdinfo always showed 0 memlock for an arena no matter how many pages it had.

Quoted 8 lines

Count the pages that are actually mapped into the arena: bump a counter in apply_range_set_cb() when a page goes in and drop it in apply_range_clear_cb() when a page goes out, both under the arena spinlock. map_mem_usage() then just returns nr_pages << PAGE_SHIFT.

Only real data pages are counted, not the scratch page.

Quoted 52 lines

---
To sashiko:
1. __bpf_alloc_page::can_alloc_pages already check whether we are under lock.
2. apply_to_page_range() does not allocate page tables here. arena_map_alloc() already did it.

---
 kernel/bpf/arena.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
kernel/bpf/arena.c +8 -2
index 8dbc24460890..f046e878f7ae 100644
@@ -55,8 +55,10 @@ struct bpf_arena {
 	struct vm_struct *kern_vm;
 	struct page *scratch_page;
 	struct range_tree rt;
-	/* protects rt */
+	/* protects rt and nr_pages */
 	rqspinlock_t spinlock;
+	/* number of pages currently populated in the arena */
+	u64 nr_pages;
 	struct list_head vma_list;
 	/* protects vma_list */
 	struct mutex lock;
@@ -196,6 +198,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
 	set_pte_at(&init_mm, addr, pte, pteval);
 #endif
 	d->i++;
+	WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1);
 	return 0;
 }
 
@@ -231,6 +234,7 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data)
 		return 0;
 
 	__llist_add(&page->pcp_llist, d->free_pages);
+	WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages - 1);
 	return 0;
 }
 
@@ -413,7 +417,9 @@ static int arena_map_check_btf(struct bpf_map *map, const struct btf *btf,
 
 static u64 arena_map_mem_usage(const struct bpf_map *map)
 {
-	return 0;
+	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
+
+	return (u64)READ_ONCE(arena->nr_pages) << PAGE_SHIFT;
 }
 
 struct vma_list {
#5 lore raw

PATCH v2 3/4 selftests/bpf: Run arena tests serially

In reply to Jiayuan Chen

Every arena reserves a 4GB kernel vmalloc region at creation, and on a KASAN build that eagerly populates ~512MB of shadow memory per arena. When several arena tests run in parallel their shadow adds up, exhausts the memory of a small CI VM and trips the OOM killer during map creation.

Make the dedicated arena tests (arena_* and libarena*) serial so they no longer pile up in the parallel phase. That alone brings the peak number of live arenas back within the CI memory budget.

A few other tests such as verifier_arena*, stream and compute_live_registers create an arena too, but they run inside shared RUN_TESTS()/RUN() suites, so turning them serial would drag many unrelated subtests along with them. Leave those as-is.

---
 tools/testing/selftests/bpf/prog_tests/arena_atomics.c      | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_direct_value.c | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_htab.c         | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_list.c         | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c    | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_strsearch.c    | 2 +-
 tools/testing/selftests/bpf/prog_tests/libarena.c           | 2 +-
 tools/testing/selftests/bpf/prog_tests/libarena_asan.c      | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)
tools/testing/selftests/bpf/prog_tests/arena_atomics.c +1 -1
index d98577a6babc..1ad5d03d07ad 100644
@@ -222,7 +222,7 @@ static void test_store_release(struct arena_atomics *skel)
 		  "store_release64_result");
 }
 
-void test_arena_atomics(void)
+void serial_test_arena_atomics(void)
 {
 	struct arena_atomics *skel;
 	int err;
tools/testing/selftests/bpf/prog_tests/arena_direct_value.c +1 -1
index 4b4adb3f4b71..01fcf4965ea4 100644
@@ -66,7 +66,7 @@ static void test_arena_direct_value_one_past_end(void)
 	close(map_fd);
 }
 
-void test_arena_direct_value(void)
+void serial_test_arena_direct_value(void)
 {
 	if (test__start_subtest("one_past_end"))
 		test_arena_direct_value_one_past_end();
tools/testing/selftests/bpf/prog_tests/arena_htab.c +1 -1
index d69fd2465f53..91ccf0402980 100644
@@ -81,7 +81,7 @@ static void test_arena_htab_asm(void)
 	arena_htab_asm__destroy(skel);
 }
 
-void test_arena_htab(void)
+void serial_test_arena_htab(void)
 {
 	if (test__start_subtest("arena_htab_llvm"))
 		test_arena_htab_llvm();
tools/testing/selftests/bpf/prog_tests/arena_list.c +1 -1
index 4f2866a615ce..2648e06f53d0 100644
@@ -68,7 +68,7 @@ static void test_arena_list_add_del(int cnt, bool nonsleepable)
 	arena_list__destroy(skel);
 }
 
-void test_arena_list(void)
+void serial_test_arena_list(void)
 {
 	if (test__start_subtest("arena_list_1"))
 		test_arena_list_add_del(1, false);
tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +1 -1
index acb9d53b5973..545b05d7a0aa 100644
@@ -101,7 +101,7 @@ static void test_arena_spin_lock_size(int size)
 	return;
 }
 
-void test_arena_spin_lock(void)
+void serial_test_arena_spin_lock(void)
 {
 	repeat = 1000;
 	if (test__start_subtest("arena_spin_lock_1"))
tools/testing/selftests/bpf/prog_tests/arena_strsearch.c +1 -1
index f81a0c066505..0c1c6cbfa0f1 100644
@@ -23,7 +23,7 @@ static void test_arena_str(void)
 	arena_strsearch__destroy(skel);
 }
 
-void test_arena_strsearch(void)
+void serial_test_arena_strsearch(void)
 {
 	if (test__start_subtest("arena_strsearch"))
 		test_arena_str();
tools/testing/selftests/bpf/prog_tests/libarena.c +1 -1
index ba5a5a50f7c0..df7e4b8dc394 100644
@@ -202,7 +202,7 @@ static void run_libarena_parallel_test(struct libarena *skel, struct bpf_program
 	run_libarena_parallel_fini(skel, name, prefixlen);
 }
 
-void test_libarena(void)
+void serial_test_libarena(void)
 {
 	struct arena_alloc_reserve_args args;
 	struct libarena *skel;
tools/testing/selftests/bpf/prog_tests/libarena_asan.c +1 -1
index f897405f701d..9c31b17dbf39 100644
@@ -85,7 +85,7 @@ static void run_test(void)
  * Run the test depending on whether LLVM can compile arena ASAN
  * programs.
  */
-void test_libarena_asan(void)
+void serial_test_libarena_asan(void)
 {
 #ifdef HAS_BPF_ARENA_ASAN
 	run_test();
Signature · 1 line
2.43.0
reviewer Reviewed-by on v2 3/4
#6 lore raw

Re: [PATCH bpf-next v2 3/4] selftests/bpf: Run arena tests serially

Quoted 9 lines · Jiayuan Chen

Every arena reserves a 4GB kernel vmalloc region at creation, and on a KASAN build that eagerly populates ~512MB of shadow memory per arena. When several arena tests run in parallel their shadow adds up, exhausts the memory of a small CI VM and trips the OOM killer during map creation.

Make the dedicated arena tests (arena_* and libarena*) serial so they no longer pile up in the parallel phase. That alone brings the peak number of live arenas back within the CI memory budget.

Quoted 6 lines

A few other tests such as verifier_arena*, stream and compute_live_registers create an arena too, but they run inside shared RUN_TESTS()/RUN() suites, so turning them serial would drag many unrelated subtests along with them. Leave those as-is.

Quoted 115 lines
---
 tools/testing/selftests/bpf/prog_tests/arena_atomics.c      | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_direct_value.c | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_htab.c         | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_list.c         | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c    | 2 +-
 tools/testing/selftests/bpf/prog_tests/arena_strsearch.c    | 2 +-
 tools/testing/selftests/bpf/prog_tests/libarena.c           | 2 +-
 tools/testing/selftests/bpf/prog_tests/libarena_asan.c      | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)
tools/testing/selftests/bpf/prog_tests/arena_atomics.c +1 -1
index d98577a6babc..1ad5d03d07ad 100644
@@ -222,7 +222,7 @@ static void test_store_release(struct arena_atomics *skel)
 		  "store_release64_result");
 }
 
-void test_arena_atomics(void)
+void serial_test_arena_atomics(void)
 {
 	struct arena_atomics *skel;
 	int err;
tools/testing/selftests/bpf/prog_tests/arena_direct_value.c +1 -1
index 4b4adb3f4b71..01fcf4965ea4 100644
@@ -66,7 +66,7 @@ static void test_arena_direct_value_one_past_end(void)
 	close(map_fd);
 }
 
-void test_arena_direct_value(void)
+void serial_test_arena_direct_value(void)
 {
 	if (test__start_subtest("one_past_end"))
 		test_arena_direct_value_one_past_end();
tools/testing/selftests/bpf/prog_tests/arena_htab.c +1 -1
index d69fd2465f53..91ccf0402980 100644
@@ -81,7 +81,7 @@ static void test_arena_htab_asm(void)
 	arena_htab_asm__destroy(skel);
 }
 
-void test_arena_htab(void)
+void serial_test_arena_htab(void)
 {
 	if (test__start_subtest("arena_htab_llvm"))
 		test_arena_htab_llvm();
tools/testing/selftests/bpf/prog_tests/arena_list.c +1 -1
index 4f2866a615ce..2648e06f53d0 100644
@@ -68,7 +68,7 @@ static void test_arena_list_add_del(int cnt, bool nonsleepable)
 	arena_list__destroy(skel);
 }
 
-void test_arena_list(void)
+void serial_test_arena_list(void)
 {
 	if (test__start_subtest("arena_list_1"))
 		test_arena_list_add_del(1, false);
tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +1 -1
index acb9d53b5973..545b05d7a0aa 100644
@@ -101,7 +101,7 @@ static void test_arena_spin_lock_size(int size)
 	return;
 }
 
-void test_arena_spin_lock(void)
+void serial_test_arena_spin_lock(void)
 {
 	repeat = 1000;
 	if (test__start_subtest("arena_spin_lock_1"))
tools/testing/selftests/bpf/prog_tests/arena_strsearch.c +1 -1
index f81a0c066505..0c1c6cbfa0f1 100644
@@ -23,7 +23,7 @@ static void test_arena_str(void)
 	arena_strsearch__destroy(skel);
 }
 
-void test_arena_strsearch(void)
+void serial_test_arena_strsearch(void)
 {
 	if (test__start_subtest("arena_strsearch"))
 		test_arena_str();
tools/testing/selftests/bpf/prog_tests/libarena.c +1 -1
index ba5a5a50f7c0..df7e4b8dc394 100644
@@ -202,7 +202,7 @@ static void run_libarena_parallel_test(struct libarena *skel, struct bpf_program
 	run_libarena_parallel_fini(skel, name, prefixlen);
 }
 
-void test_libarena(void)
+void serial_test_libarena(void)
 {
 	struct arena_alloc_reserve_args args;
 	struct libarena *skel;
tools/testing/selftests/bpf/prog_tests/libarena_asan.c +1 -1
index f897405f701d..9c31b17dbf39 100644
@@ -85,7 +85,7 @@ static void run_test(void)
  * Run the test depending on whether LLVM can compile arena ASAN
  * programs.
  */
-void test_libarena_asan(void)
+void serial_test_libarena_asan(void)
 {
 #ifdef HAS_BPF_ARENA_ASAN
 	run_test();
#7 lore raw

PATCH v2 4/4 selftests/bpf: Add tests for memory usage for arena

In reply to Jiayuan Chen

Allocate and free arena pages, both from BPF and via user-space fault-in, and check that the map's memlock in fdinfo tracks the number of pages that are actually populated.

Like the other arena tests it runs serially.

test:
./test_progs -a arena_mem_usage
#5 arena_mem_usage:OK
Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED

---
 .../bpf/prog_tests/arena_mem_usage.c          | 122 ++++++++++++++++++
 .../selftests/bpf/progs/arena_mem_usage.c     |  40 ++++++
 2 files changed, 162 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_mem_usage.c
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +122 -0
new file mode 100644
index 000000000000..14c2d1a1d673
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include <sys/user.h>
+#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */
+#include <unistd.h>
+#define PAGE_SIZE getpagesize()
+#endif
+
+#include "arena_mem_usage.skel.h"
+
+/*
+ * arena_map_mem_usage() is surfaced to user space through the map's
+ * /proc/<pid>/fdinfo/<fd> "memlock:" line (the same value bpftool map show
+ * prints). Read it directly so the test has no external dependency.
+ */
+static long map_memlock(int map_fd)
+{
+	char path[64], line[128];
+	long memlock = -1;
+	FILE *f;
+
+	snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", map_fd);
+	f = fopen(path, "r");
+	if (!ASSERT_OK_PTR(f, "open_fdinfo"))
+		return -1;
+	while (fgets(line, sizeof(line), f)) {
+		if (sscanf(line, "memlock:\t%ld", &memlock) == 1)
+			break;
+	}
+	fclose(f);
+	ASSERT_NEQ(memlock, -1, "parse_memlock");
+	return memlock;
+}
+
+static int run(struct bpf_program *prog, const char *name)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	int err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
+
+	if (!ASSERT_OK(err, name))
+		return -1;
+	if (!ASSERT_OK(opts.retval, name))
+		return -1;
+	return 0;
+}
+
+void serial_test_arena_mem_usage(void)
+{
+	struct arena_mem_usage *skel;
+	const long ps = PAGE_SIZE;
+	char *base;
+	size_t sz;
+	int fd, i;
+
+	skel = arena_mem_usage__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_load"))
+		return;
+	fd = bpf_map__fd(skel->maps.arena);
+
+	/* Fresh arena: no data pages, and the scratch page is not counted. */
+	ASSERT_EQ(map_memlock(fd), 0, "initial");
+
+	/* BPF-side allocation of 17 pages. */
+	skel->bss->alloc_cnt = 17;
+	if (run(skel->progs.alloc, "alloc"))
+		goto out;
+	/*
+	 * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host
+	 * is under memory pressure), not a miscount -- flag it distinctly so a
+	 * red CI run is not mistaken for a counting bug.
+	 */
+	if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 17 * ps, "after_alloc");
+
+	/* Free a single page (arena_free_pages page_cnt==1 path). */
+	skel->bss->free_byte_off = 0;
+	skel->bss->free_cnt = 1;
+	if (run(skel->progs.free_pages, "free_one"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 16 * ps, "after_free_one");
+
+	/* Free ten pages in one call (bulk path); only the freed pages count. */
+	skel->bss->free_byte_off = 1 * ps;
+	skel->bss->free_cnt = 10;
+	if (run(skel->progs.free_pages, "free_bulk"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 6 * ps, "after_free_bulk");
+
+	/* Free the remaining six -> arena empty again. */
+	skel->bss->free_byte_off = 11 * ps;
+	skel->bss->free_cnt = 6;
+	if (run(skel->progs.free_pages, "free_rest"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 0, "after_free_rest");
+
+	/*
+	 * User-space fault-in: touching unallocated arena pages allocates them
+	 * through arena_vm_fault(). libbpf mmap()s the arena at map_extra during
+	 * load, so bpf_map__initial_value() hands back that base.
+	 */
+	base = bpf_map__initial_value(skel->maps.arena, &sz);
+	if (!ASSERT_OK_PTR(base, "arena_base"))
+		goto out;
+	for (i = 0; i < 8; i++)
+		base[i * ps] = 1;
+	ASSERT_EQ(map_memlock(fd), 8 * ps, "after_faultin");
+
+	/*
+	 * Free the faulted-in pages from BPF. They are mapped into the user vma
+	 * (elevated refcount), so this also exercises the zap path.
+	 */
+	skel->bss->ptr = base;
+	skel->bss->free_byte_off = 0;
+	skel->bss->free_cnt = 8;
+	if (run(skel->progs.free_pages, "free_faulted"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 0, "after_free_faulted");
+out:
+	arena_mem_usage__destroy(skel);
+}
tools/testing/selftests/bpf/progs/arena_mem_usage.c new file +40 -0
new file mode 100644
index 000000000000..455ecd669a5a
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_arena_common.h"
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARENA);
+	__uint(map_flags, BPF_F_MMAPABLE);
+	__uint(max_entries, 1000); /* number of pages */
+#ifdef __TARGET_ARCH_arm64
+	__ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
+#else
+	__ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
+#endif
+} arena SEC(".maps");
+
+void __arena *ptr;
+int alloc_cnt;		/* in:  pages to allocate */
+long free_byte_off;	/* in:  byte offset within ptr to start freeing */
+int free_cnt;		/* in:  pages to free */
+
+SEC("syscall")
+int alloc(void *ctx)
+{
+	ptr = bpf_arena_alloc_pages(&arena, NULL, alloc_cnt, NUMA_NO_NODE, 0);
+	/* Success/failure is checked from user space via skel->bss->ptr. */
+	return 0;
+}
+
+SEC("syscall")
+int free_pages(void *ctx)
+{
+	if (!ptr)
+		return 1;
+	bpf_arena_free_pages(&arena, (char __arena *)ptr + free_byte_off, free_cnt);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
Signature · 1 line
2.43.0
#8 lore raw

Re: [PATCH bpf-next v2 4/4] selftests/bpf: Add tests for memory usage for arena

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [Medium] Incorrect use of ASSERT_OK_PTR to check for a NULL pointer read from BPF userspace memory…

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] Incorrect use of ASSERT_OK_PTR to check for a NULL pointer read from BPF userspace memory, breaking intended error handling. --

--- Patch [4]: [PATCH bpf-next v2 4/4] selftests/bpf: Add tests for memory usage for arena --- commit b8bd79585d9f7cbf2a83d5381d0df2c316e65ed7
Author: Jiayuan Chen <[email protected]>

selftests/bpf: Add tests for memory usage for arena

This patch adds a selftest to verify memory usage tracking for BPF arenas. It allocates and frees arena pages both from BPF and user-space and checks the map's memlock value via fdinfo to ensure it tracks populated pages.

Quoted 5 lines
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +0 -0
new file mode 100644
index 0000000000000..14c2d1a1d673c

[ ... ]

Quoted 16 lines
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +16 -0
+void serial_test_arena_mem_usage(void)
+{
+	struct arena_mem_usage *skel;
+	const long ps = PAGE_SIZE;
+	char *base;
+	size_t sz;
+	int fd, i;
+
+	skel = arena_mem_usage__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_load"))
+		return;
+	fd = bpf_map__fd(skel->maps.arena);
+
+	/* Fresh arena: no data pages, and the scratch page is not counted. */
+	ASSERT_EQ(map_memlock(fd), 0, "initial");
+

tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c line 73

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +10 -0
+	/* BPF-side allocation of 17 pages. */
+	skel->bss->alloc_cnt = 17;
+	if (run(skel->progs.alloc, "alloc"))
+		goto out;
+	/*
+	 * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host
+	 * is under memory pressure), not a miscount -- flag it distinctly so a
+	 * red CI run is not mistaken for a counting bug.
+	 */
+	if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages"))

[Severity: Medium]
Will this check actually catch a NULL pointer when bpf_arena_alloc_pages() fails?

When skel->bss->ptr is NULL, ASSERT_OK_PTR() evaluates libbpf_get_error(NULL), which returns -errno. Since the userspace errno is likely 0 following a successful bpf_prog_test_run_opts() ioctl call, ASSERT_OK_PTR silently evaluates to true (success).

This would cause the goto out to be skipped, and the test to proceed to evaluate ASSERT_EQ(map_memlock(fd), 17 * ps). This subsequent check would then fail, emitting the exact counting bug error the comment mentions trying to avoid.

Should this use a different macro, like ASSERT_NEQ(skel->bss->ptr, NULL), to correctly verify the userspace pointer?

Quoted 2 lines
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +2 -0
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 17 * ps, "after_alloc");

[ ... ]

Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=4
#9 lore raw

Re: [PATCH bpf-next v2 4/4] selftests/bpf: Add tests for memory usage for arena

Quoted 56 lines · On 7/17/26 7:53 PM, [email protected] wrote:

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] Incorrect use of ASSERT_OK_PTR to check for a NULL pointer read from BPF userspace memory, breaking intended error handling. --

--- Patch [4]: [PATCH bpf-next v2 4/4] selftests/bpf: Add tests for memory usage for arena --- commit b8bd79585d9f7cbf2a83d5381d0df2c316e65ed7
Author: Jiayuan Chen <[email protected]>

selftests/bpf: Add tests for memory usage for arena

This patch adds a selftest to verify memory usage tracking for BPF arenas. It allocates and frees arena pages both from BPF and user-space and checks the map's memlock value via fdinfo to ensure it tracks populated pages.

Quoted 5 lines
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +0 -0
new file mode 100644
index 0000000000000..14c2d1a1d673c

[ ... ]

Quoted 16 lines
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +16 -0
+void serial_test_arena_mem_usage(void)
+{
+	struct arena_mem_usage *skel;
+	const long ps = PAGE_SIZE;
+	char *base;
+	size_t sz;
+	int fd, i;
+
+	skel = arena_mem_usage__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_load"))
+		return;
+	fd = bpf_map__fd(skel->maps.arena);
+
+	/* Fresh arena: no data pages, and the scratch page is not counted. */
+	ASSERT_EQ(map_memlock(fd), 0, "initial");
+

tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c line 73

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +10 -0
+	/* BPF-side allocation of 17 pages. */
+	skel->bss->alloc_cnt = 17;
+	if (run(skel->progs.alloc, "alloc"))
+		goto out;
+	/*
+	 * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host
+	 * is under memory pressure), not a miscount -- flag it distinctly so a
+	 * red CI run is not mistaken for a counting bug.
+	 */
+	if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages"))

[Severity: Medium]
Will this check actually catch a NULL pointer when bpf_arena_alloc_pages() fails?

When skel->bss->ptr is NULL, ASSERT_OK_PTR() evaluates libbpf_get_error(NULL), which returns -errno. Since the userspace errno is likely 0 following a successful bpf_prog_test_run_opts() ioctl call, ASSERT_OK_PTR silently evaluates to true (success).

Quoted 7 lines

This would cause the goto out to be skipped, and the test to proceed to evaluate ASSERT_EQ(map_memlock(fd), 17 * ps). This subsequent check would then fail, emitting the exact counting bug error the comment mentions trying to avoid.

Should this use a different macro, like ASSERT_NEQ(skel->bss->ptr, NULL), to correctly verify the userspace pointer?

On success ptr is a valid arena address and ASSERT_OK_PTR() passes correctly.

On alloc failure ptr is NULL and ASSERT_OK_PTR() may treat it as OK (libbpf_get_error(NULL) returns -errno, which can be 0), but the next ASSERT_EQ(map_memlock(fd), 17 * ps) still fails.

I can switch it to ASSERT_NEQ(skel->bss->ptr, NULL, ...) in the next revision.

reviewer Reviewed-by on v2 4/4
#10 lore raw

Re: [PATCH bpf-next v2 4/4] selftests/bpf: Add tests for memory usage for arena

In reply to Jiayuan Chen
Quoted 4 lines · Jiayuan Chen

Allocate and free arena pages, both from BPF and via user-space fault-in, and check that the map's memlock in fdinfo tracks the number of pages that are actually populated.

Quoted 8 lines

Like the other arena tests it runs serially.

test:
./test_progs -a arena_mem_usage
#5 arena_mem_usage:OK
Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED

The Sashiko nit is worth addressing, but overall looks fine.

Quoted 181 lines
---
 .../bpf/prog_tests/arena_mem_usage.c          | 122 ++++++++++++++++++
 .../selftests/bpf/progs/arena_mem_usage.c     |  40 ++++++
 2 files changed, 162 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_mem_usage.c
tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file +122 -0
new file mode 100644
index 000000000000..14c2d1a1d673
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include <sys/user.h>
+#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */
+#include <unistd.h>
+#define PAGE_SIZE getpagesize()
+#endif
+
+#include "arena_mem_usage.skel.h"
+
+/*
+ * arena_map_mem_usage() is surfaced to user space through the map's
+ * /proc/<pid>/fdinfo/<fd> "memlock:" line (the same value bpftool map show
+ * prints). Read it directly so the test has no external dependency.
+ */
+static long map_memlock(int map_fd)
+{
+	char path[64], line[128];
+	long memlock = -1;
+	FILE *f;
+
+	snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", map_fd);
+	f = fopen(path, "r");
+	if (!ASSERT_OK_PTR(f, "open_fdinfo"))
+		return -1;
+	while (fgets(line, sizeof(line), f)) {
+		if (sscanf(line, "memlock:\t%ld", &memlock) == 1)
+			break;
+	}
+	fclose(f);
+	ASSERT_NEQ(memlock, -1, "parse_memlock");
+	return memlock;
+}
+
+static int run(struct bpf_program *prog, const char *name)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	int err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
+
+	if (!ASSERT_OK(err, name))
+		return -1;
+	if (!ASSERT_OK(opts.retval, name))
+		return -1;
+	return 0;
+}
+
+void serial_test_arena_mem_usage(void)
+{
+	struct arena_mem_usage *skel;
+	const long ps = PAGE_SIZE;
+	char *base;
+	size_t sz;
+	int fd, i;
+
+	skel = arena_mem_usage__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_load"))
+		return;
+	fd = bpf_map__fd(skel->maps.arena);
+
+	/* Fresh arena: no data pages, and the scratch page is not counted. */
+	ASSERT_EQ(map_memlock(fd), 0, "initial");
+
+	/* BPF-side allocation of 17 pages. */
+	skel->bss->alloc_cnt = 17;
+	if (run(skel->progs.alloc, "alloc"))
+		goto out;
+	/*
+	 * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host
+	 * is under memory pressure), not a miscount -- flag it distinctly so a
+	 * red CI run is not mistaken for a counting bug.
+	 */
+	if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 17 * ps, "after_alloc");
+
+	/* Free a single page (arena_free_pages page_cnt==1 path). */
+	skel->bss->free_byte_off = 0;
+	skel->bss->free_cnt = 1;
+	if (run(skel->progs.free_pages, "free_one"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 16 * ps, "after_free_one");
+
+	/* Free ten pages in one call (bulk path); only the freed pages count. */
+	skel->bss->free_byte_off = 1 * ps;
+	skel->bss->free_cnt = 10;
+	if (run(skel->progs.free_pages, "free_bulk"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 6 * ps, "after_free_bulk");
+
+	/* Free the remaining six -> arena empty again. */
+	skel->bss->free_byte_off = 11 * ps;
+	skel->bss->free_cnt = 6;
+	if (run(skel->progs.free_pages, "free_rest"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 0, "after_free_rest");
+
+	/*
+	 * User-space fault-in: touching unallocated arena pages allocates them
+	 * through arena_vm_fault(). libbpf mmap()s the arena at map_extra during
+	 * load, so bpf_map__initial_value() hands back that base.
+	 */
+	base = bpf_map__initial_value(skel->maps.arena, &sz);
+	if (!ASSERT_OK_PTR(base, "arena_base"))
+		goto out;
+	for (i = 0; i < 8; i++)
+		base[i * ps] = 1;
+	ASSERT_EQ(map_memlock(fd), 8 * ps, "after_faultin");
+
+	/*
+	 * Free the faulted-in pages from BPF. They are mapped into the user vma
+	 * (elevated refcount), so this also exercises the zap path.
+	 */
+	skel->bss->ptr = base;
+	skel->bss->free_byte_off = 0;
+	skel->bss->free_cnt = 8;
+	if (run(skel->progs.free_pages, "free_faulted"))
+		goto out;
+	ASSERT_EQ(map_memlock(fd), 0, "after_free_faulted");
+out:
+	arena_mem_usage__destroy(skel);
+}
tools/testing/selftests/bpf/progs/arena_mem_usage.c new file +40 -0
new file mode 100644
index 000000000000..455ecd669a5a
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_arena_common.h"
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARENA);
+	__uint(map_flags, BPF_F_MMAPABLE);
+	__uint(max_entries, 1000); /* number of pages */
+#ifdef __TARGET_ARCH_arm64
+	__ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
+#else
+	__ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
+#endif
+} arena SEC(".maps");
+
+void __arena *ptr;
+int alloc_cnt;		/* in:  pages to allocate */
+long free_byte_off;	/* in:  byte offset within ptr to start freeing */
+int free_cnt;		/* in:  pages to free */
+
+SEC("syscall")
+int alloc(void *ctx)
+{
+	ptr = bpf_arena_alloc_pages(&arena, NULL, alloc_cnt, NUMA_NO_NODE, 0);
+	/* Success/failure is checked from user space via skel->bss->ptr. */
+	return 0;
+}
+
+SEC("syscall")
+int free_pages(void *ctx)
+{
+	if (!ptr)
+		return 1;
+	bpf_arena_free_pages(&arena, (char __arena *)ptr + free_byte_off, free_cnt);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
#11 lore raw
In reply to Jiayuan Chen
This series was applied to bpf/bpf-next.git (master) by Kumar Kartikeya Dwivedi <[email protected]>:

Hello:

This series was applied to bpf/bpf-next.git (master) by Kumar Kartikeya Dwivedi <[email protected]>:

Quoted 5 lines · On Fri, 17 Jul 2026 19:37:00 +0800 you wrote:

arena is the only map type whose map_mem_usage() still returns 0, so bpftool and fdinfo always show 0 memlock for it no matter how many pages it holds. This series adds the accounting, runs the arena selftests serially so they don't OOM a small CI VM, and adds a test for it.

Quoted 8 lines

v1 -> v2:
- split the scratch_page -> arena refactor into a prep patch (no functional change)
- use WRITE_ONCE() for nr_pages to pair with the lockless reader
- reword the "run serially" commit message
- selftest: alloc prog returns 0 so the failure check is reached

[...]

Here is the summary with links:
- [bpf-next,v2,1/4] bpf: Pass arena instead of scratch_page to the pte callbacks

    https://git.kernel.org/bpf/bpf-next/c/f8248ac8f044
  - [bpf-next,v2,2/4] bpf: Add memory usage for arena
    https://git.kernel.org/bpf/bpf-next/c/89318afb1414
  - [bpf-next,v2,3/4] selftests/bpf: Run arena tests serially
    https://git.kernel.org/bpf/bpf-next/c/a14f9ba080c1
  - [bpf-next,v2,4/4] selftests/bpf: Add tests for memory usage for arena
    https://git.kernel.org/bpf/bpf-next/c/29e4bcf12604

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html