mm: huge_memory: a new debugfs interface for splitting THP tests.
Subsystems (3)
PATCH v7 1/2 mm: huge_memory: a new debugfs interface for splitting THP tests.
From: Zi Yan <[email protected]>
We did not have a direct user interface of splitting the compound page backing a THP and there is no need unless we want to expose the THP implementation details to users. Make <debugfs>/split_huge_pages accept a new command to do that.
By writing "<pid>,<vaddr_start>,<vaddr_end>" to <debugfs>/split_huge_pages, THPs within the given virtual address range from the process with the given pid are split. It is used to test split_huge_page function. In addition, a selftest program is added to tools/testing/selftests/vm to utilize the interface by splitting PMD THPs and PTE-mapped THPs.
This does not change the old behavior, i.e., writing 1 to the interface to split all THPs in the system.
Changelog:
From v6:
1. pr_info -> pr_debug.
2. Added cond_resched() in all split loops. (suggested by David Rientjes)
From v5:
1. Skipped special VMAs and other fixes. (suggested by Yang Shi)
From v4:
1. Fixed the error code return issue, spotted by kernel test robot <[email protected]>.
From v3:
1. Factored out split huge pages in the given pid code to a separate function.
2. Added the missing put_page for not split pages.
3. pr_debug -> pr_info, make reading results simpler.
From v2:
1. Reused existing <debugfs>/split_huge_pages interface. (suggested by Yang Shi)
From v1:
1. Removed unnecessary calling to vma_migratable, spotted by kernel test robot <[email protected]>.
2. Dropped the use of find_mm_struct and code it directly, since there is no need for the permission check in that function and the function is only available when migration is on.
3. Added some comments in the selftest program to clarify how PTE-mapped THPs are formed.
--- mm/huge_memory.c | 155 ++++++++- tools/testing/selftests/vm/.gitignore | 1 + tools/testing/selftests/vm/Makefile | 1 + .../selftests/vm/split_huge_page_test.c | 318 ++++++++++++++++++ 4 files changed, 467 insertions(+), 8 deletions(-) create mode 100644 tools/testing/selftests/vm/split_huge_page_test.c
mm/huge_memory.c +147 -8
@@ -7,6 +7,7 @@#include <linux/mm.h> #include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/sched/coredump.h> #include <linux/sched/numa_balancing.h> #include <linux/highmem.h>
@@ -2927,16 +2928,14 @@ static struct shrinker deferred_split_shrinker = { };#ifdef CONFIG_DEBUG_FS -static int split_huge_pages_set (void *data,u64 val) +static void split_huge_pages_all (void ) {struct zone *zone;struct page *page;unsigned long pfn, max_zone_pfn;unsigned long total =0 , split =0 ; -if (val !=1 ) -return -EINVAL ; - +pr_debug ("Split all THPs\n" );for_each_populated_zone (zone) { max_zone_pfn =zone_end_pfn (zone);for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
@@ -2960,15 +2959,155 @@ static int split_huge_pages_set(void *data, u64 val)unlock_page (page); next:put_page (page); +cond_resched (); } } -pr_info ("%lu of %lu THP split\n" , split, total); +pr_debug ("%lu of %lu THP split\n" , split, total); +} -return 0 ; +static inline bool vma_not_suitable_for_thp_split (struct vm_area_struct *vma) +{ +return vma_is_special_huge (vma) || (vma->vm_flags &VM_IO ) || +is_vm_hugetlb_page (vma); +} + +static int split_huge_pages_pid (int pid,unsigned long vaddr_start, +unsigned long vaddr_end) +{ +int ret =0 ; +struct task_struct *task; +struct mm_struct *mm; +unsigned long total =0 , split =0 ; +unsigned long addr; + + vaddr_start &=PAGE_MASK ; + vaddr_end &=PAGE_MASK ; + +/* Find the task_struct from pid */ +rcu_read_lock (); + task =find_task_by_vpid (pid); +if (!task) { +rcu_read_unlock (); + ret = -ESRCH ; + goto out; + } +get_task_struct (task); +rcu_read_unlock (); + +/* Find the mm_struct */ + mm =get_task_mm (task); +put_task_struct (task); + +if (!mm) { + ret = -EINVAL ; + goto out; + } + +pr_debug ("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n" , + pid, vaddr_start, vaddr_end); + +mmap_read_lock (mm); +/* +* always increase addr by PAGE_SIZE, since we could have a PTE page +* table filled with PTE-mapped THPs, each of which is distinct. +*/ +for (addr = vaddr_start; addr < vaddr_end; addr +=PAGE_SIZE ) { +struct vm_area_struct *vma =find_vma (mm, addr); +unsigned int follflags; +struct page *page; + +if (!vma || addr < vma->vm_start) +break ; + +/* skip special VMA and hugetlb VMA */ +if (vma_not_suitable_for_thp_split (vma)) { + addr = vma->vm_end; +continue ; + } + +/* FOLL_DUMP to ignore special (like zero) pages */ + follflags =FOLL_GET |FOLL_DUMP ; + page =follow_page (vma, addr, follflags); + +if (IS_ERR (page)) +continue ; +if (!page) +continue ; + +if (!is_transparent_hugepage (page)) + goto next; + + total++; +if (!can_split_huge_page (compound_head (page),NULL )) + goto next; + +if (!trylock_page (page)) + goto next; + +if (!split_huge_page (page)) + split++; + +unlock_page (page); +next: +put_page (page); +cond_resched (); + } +mmap_read_unlock (mm); +mmput (mm); + +pr_debug ("%lu of %lu THP split\n" , split, total); + +out: +return ret; } -DEFINE_DEBUGFS_ATTRIBUTE (split_huge_pages_fops,NULL , split_huge_pages_set, -"%llu\n" ); + +#define MAX_INPUT_BUF_SZ 255 + +static ssize_t split_huge_pages_write (struct file *file,const char __user *buf, +size_t count,loff_t *ppops) +{ +static DEFINE_MUTEX (split_debug_mutex); +ssize_t ret; +char input_buf[MAX_INPUT_BUF_SZ ];/* hold pid, start_vaddr, end_vaddr */ +int pid; +unsigned long vaddr_start, vaddr_end; + + ret =mutex_lock_interruptible (&split_debug_mutex); +if (ret) +return ret; + + ret = -EFAULT ; + +memset (input_buf,0 ,MAX_INPUT_BUF_SZ ); +if (copy_from_user (input_buf, buf,min_t (size_t, count,MAX_INPUT_BUF_SZ ))) + goto out; + + input_buf[MAX_INPUT_BUF_SZ -1 ] ='\0' ; + ret =sscanf (input_buf,"%d,0x%lx,0x%lx" , &pid, &vaddr_start, &vaddr_end); +if (ret ==1 && pid ==1 ) { +split_huge_pages_all (); + ret =strlen (input_buf); + goto out; + }else if (ret !=3 ) { + ret = -EINVAL ; + goto out; + } + + ret =split_huge_pages_pid (pid, vaddr_start, vaddr_end); +if (!ret) + ret =strlen (input_buf); +out: +mutex_unlock (&split_debug_mutex); +return ret; + +} + +static const struct file_operations split_huge_pages_fops = { + .owner =THIS_MODULE , + .write = split_huge_pages_write, + .llseek = no_llseek, +};static int __init split_huge_pages_debugfs (void ) {
tools/testing/selftests/vm/.gitignore +1 -0
@@ -23,3 +23,4 @@ write_to_hugetlbfs hmm-tests memfd_secret local_config.* +split_huge_page_test
tools/testing/selftests/vm/Makefile +1 -0
@@ -43,6 +43,7 @@ TEST_GEN_FILES += on-fault-limit TEST_GEN_FILES += thuge-gen TEST_GEN_FILES += transhuge-stress TEST_GEN_FILES += userfaultfd +TEST_GEN_FILES += split_huge_page_test ifeq ($(MACHINE),x86_64) CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh $(CC) ../x86/trivial_32bit_program.c -m32)
tools/testing/selftests/vm/split_huge_page_test.c new file +318 -0
318 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH v7 2/2 mm: huge_memory: debugfs for file-backed THP split.
From: Zi Yan <[email protected]>
Further extend <debugfs>/split_huge_pages to accept "<path>,<off_start>,<off_end>" for file-backed THP split tests since tmpfs may have file backed by THP that mapped nowhere.
Update selftest program to test file-backed THP split too.
--- mm/huge_memory.c | 91 ++++++++++++++++++- .../selftests/vm/split_huge_page_test.c | 81 ++++++++++++++++- 2 files changed, 166 insertions(+), 6 deletions(-)
mm/huge_memory.c +90 -1
@@ -3062,6 +3062,66 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start,return ret; } +static int split_huge_pages_in_file (const char *file_path,pgoff_t off_start, +pgoff_t off_end) +{ +struct filename *file; +struct file *candidate; +struct address_space *mapping; +int ret = -EINVAL ; +pgoff_t off_cur; +unsigned long total =0 , split =0 ; + + file =getname_kernel (file_path); +if (IS_ERR (file)) +return ret; + + candidate =file_open_name (file,O_RDONLY ,0 ); +if (IS_ERR (candidate)) + goto out; + +pr_debug ("split file-backed THPs in file: %s, offset: [0x%lx - 0x%lx]\n" , + file_path, off_start, off_end); + + mapping = candidate->f_mapping; + +for (off_cur = off_start; off_cur < off_end;) { +struct page *fpage =pagecache_get_page (mapping, off_cur, +FGP_ENTRY |FGP_HEAD ,0 ); + +if (xa_is_value (fpage) || !fpage) { + off_cur +=PAGE_SIZE ; +continue ; + } + +if (!is_transparent_hugepage (fpage)) { + off_cur +=PAGE_SIZE ; + goto next; + } + total++; + off_cur = fpage->index +thp_size (fpage); + +if (!trylock_page (fpage)) + goto next; + +if (!split_huge_page (fpage)) + split++; + +unlock_page (fpage); +next: +put_page (fpage); +cond_resched (); + } + +filp_close (candidate,NULL ); + ret =0 ; + +pr_debug ("%lu of %lu file-backed THP split\n" , split, total); +out: +putname (file); +return ret; +} +#define MAX_INPUT_BUF_SZ 255 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf,
@@ -3069,7 +3129,8 @@ static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, {static DEFINE_MUTEX (split_debug_mutex);ssize_t ret; -char input_buf[MAX_INPUT_BUF_SZ ];/* hold pid, start_vaddr, end_vaddr */ +/* hold pid, start_vaddr, end_vaddr or file_path, off_start, off_end */ +char input_buf[MAX_INPUT_BUF_SZ ];int pid;unsigned long vaddr_start, vaddr_end;
@@ -3084,6 +3145,34 @@ static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, goto out; input_buf[MAX_INPUT_BUF_SZ -1 ] ='\0' ; + +if (input_buf[0 ] =='/' ) { +char *tok; +char *buf = input_buf; +char file_path[MAX_INPUT_BUF_SZ ]; +pgoff_t off_start =0 , off_end =0 ; +size_t input_len =strlen (input_buf); + + tok =strsep (&buf,"," ); +if (tok) { +strncpy (file_path, tok,MAX_INPUT_BUF_SZ ); + }else { + ret = -EINVAL ; + goto out; + } + + ret =sscanf (buf,"0x%lx,0x%lx" , &off_start, &off_end); +if (ret !=2 ) { + ret = -EINVAL ; + goto out; + } + ret =split_huge_pages_in_file (file_path, off_start, off_end); +if (!ret) + ret = input_len; + + goto out; + } + ret =sscanf (input_buf,"%d,0x%lx,0x%lx" , &pid, &vaddr_start, &vaddr_end);if (ret ==1 && pid ==1 ) {split_huge_pages_all ();
tools/testing/selftests/vm/split_huge_page_test.c +76 -5
132 lines not on this page · raw message
Signature · 1 line
2.30.2
Re: [PATCH v7 2/2] mm: huge_memory: debugfs for file-backed THP split.
Quoted 5 lines · Zi Yan
mm/huge_memory.c +5
-0+for (off_cur = off_start; off_cur < off_end;) { +struct page *fpage =pagecache_get_page (mapping, off_cur, +FGP_ENTRY |FGP_HEAD ,0 ); + +if (xa_is_value (fpage) || !fpage) {
mm/huge_memory.c static int split_huge_pages_pid(int pid, unsigned long vaddr_start, · line 3102
Quoted 10 lines
mm/huge_memory.c +10
-0+ off_cur +=PAGE_SIZE ; +continue ; + } + +if (!is_transparent_hugepage (fpage)) { + off_cur +=PAGE_SIZE ; + goto next; + } + total++; + off_cur = fpage->index +thp_size (fpage);
That can't be right. fpage->index is in units of pages and thp_size is in units of bytes. I wish C had a better type system.
I think you meant:
off_cur = fpage->index + thp_nr_pages(fpage);
Also, I think this loop would read better as ...
for (index = off_start; index < off_end; index += nr_pages) {
struct page *fpage = pagecache_get_page(mapping, index,
FGP_ENTRY | FGP_HEAD, 0);
nr_pages = 1;
if (xa_is_value(fpage) || !fpage)
continue;
if (!is_transparent_hugepage(fpage))
goto next;
total++;
nr_pages = thp_nr_pages(fpage);...
Zi Yan
reviewerRe: [PATCH v7 2/2] mm: huge_memory: debugfs for file-backed THP split.
Quoted 35 lines · Matthew Wilcox
Quoted 5 lines · Zi Yan
mm/huge_memory.c +5
-0+for (off_cur = off_start; off_cur < off_end;) { +struct page *fpage =pagecache_get_page (mapping, off_cur, +FGP_ENTRY |FGP_HEAD ,0 ); + +if (xa_is_value (fpage) || !fpage) {mm/huge_memory.c static int split_huge_pages_pid(int pid, unsigned long vaddr_start, · line 3102
Quoted 10 lines
mm/huge_memory.c +10
-0+ off_cur +=PAGE_SIZE ; +continue ; + } + +if (!is_transparent_hugepage (fpage)) { + off_cur +=PAGE_SIZE ; + goto next; + } + total++; + off_cur = fpage->index +thp_size (fpage);That can't be right.
fpage->indexis in units of pages andthp_sizeis in units of bytes. I wish C had a better type system.
I think you meant:off_cur = fpage->index + thp_nr_pages(fpage);Also, I think this loop would read better as ...
for (index = off_start; index < off_end; index += nr_pages) { struct page *fpage = pagecache_get_page(mapping, index, FGP_ENTRY | FGP_HEAD, 0); nr_pages = 1; if (xa_is_value(fpage) || !fpage) continue; if (!is_transparent_hugepage(fpage)) goto next; total++; nr_pages = thp_nr_pages(fpage);
Quoted 1 line
...
Thanks for catching this! I mixed this with looping through VMA, which is in units of bytes. I will fix this and use your suggested loop code.
—
Best Regards,
Yan Zi