mm/hugetlb: code refine and simplification
Subsystems (2)
v4 cover mm/hugetlb: code refine and simplification
Following are some cleanup for hugetlb.
Simple test with tools/testing/selftests/vm/map_hugetlb pass.
v4:
* fix a logic error for patch 7, thanks Mike
v3:
* rebase on v5.9-rc2 which adjust the last patch a little
v2:
* drop 5/6/10 since similar patches are merged or under review.
* adjust 2 based on comment from Mike Kravetz
Wei Yang (7):
mm/hugetlb: not necessary to coalesce regions recursively mm/hugetlb: remove VM_BUG_ON(!nrg) in
get_file_region_entry_from_cache()
mm/hugetlb: use list_splice to merge two list at once mm/hugetlb: count file_region to be added when regions_needed != NULL mm/hugetlb: a page from buddy is not on any list mm/hugetlb: narrow the hugetlb_lock protection area during preparing
huge page
mm/hugetlb: take the free hpage during the iteration directly
mm/hugetlb.c | 73 ++++++++++++++++++++++------------------------------ 1 file changed, 31 insertions(+), 42 deletions(-)
Signature · 1 line
2.20.1 (Apple Git-117)
PATCH v4 1/7 mm/hugetlb: not necessary to coalesce regions recursively
Per my understanding, we keep the regions ordered and would always coalesce regions properly. So the task to keep this property is just to coalesce its neighbour.
Let's simplify this.
--- mm/hugetlb.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
mm/hugetlb.c +1 -5
@@ -309,8 +309,7 @@ static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)list_del (&rg->link);kfree (rg); -coalesce_file_region (resv, prg); -return ; + rg = prg; } nrg =list_next_entry (rg, link);
@@ -320,9 +319,6 @@ static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)list_del (&rg->link);kfree (rg); - -coalesce_file_region (resv, nrg); -return ; } }
Signature · 1 line
2.20.1 (Apple Git-117)
PATCH v4 2/7 mm/hugetlb: remove VM_BUG_ON(!nrg) in get_file_region_entry_from_cache()
In reply to Wei YangWe are sure to get a valid file_region, otherwise the VM_BUG_ON(resv->region_cache_count <= 0) at the very beginning would be triggered.
Let's remove the redundant one.
--- mm/hugetlb.c | 1 - 1 file changed, 1 deletion(-)
mm/hugetlb.c +0 -1
@@ -240,7 +240,6 @@ get_file_region_entry_from_cache(struct resv_map *resv, long from, long to) resv->region_cache_count--; nrg = list_first_entry(&resv->region_cache, struct file_region, link); -VM_BUG_ON (!nrg);list_del (&nrg->link); nrg->from = from;
Signature · 1 line
2.20.1 (Apple Git-117)
PATCH v4 3/7 mm/hugetlb: use list_splice to merge two list at once
In reply to Wei YangInstead of add allocated file_region one by one to region_cache, we could use list_splice to merge two list at once.
Also we know the number of entries in the list, increase the number directly.
--- mm/hugetlb.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)
mm/hugetlb.c +2 -5
@@ -443,11 +443,8 @@ static int allocate_file_region_entries(struct resv_map *resv,spin_lock (&resv->lock); -list_for_each_entry_safe (rg, trg, &allocated_regions, link) { -list_del (&rg->link); -list_add (&rg->link, &resv->region_cache); - resv->region_cache_count++; - } +list_splice (&allocated_regions, &resv->region_cache); + resv->region_cache_count += to_allocate; }return 0 ;
Signature · 1 line
2.20.1 (Apple Git-117)
PATCH v4 4/7 mm/hugetlb: count file_region to be added when regions_needed != NULL
In reply to Wei YangThere are only two cases of function add_reservation_in_range()
* count file_region and return the number in regions_needed
* do the real list operation without counting
This means it is not necessary to have two parameters to classify these two cases.
Just use regions_needed to separate them.
--- mm/hugetlb.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)
mm/hugetlb.c +17 -16
@@ -321,16 +321,17 @@ static void coalesce_file_region(struct resv_map *resv, struct file_region *rg) } } -/* Must be called with resv->lock held. Calling this with count_only == true -* will count the number of pages to be added but will not modify the linked -* list. If regions_needed != NULL and count_only == true, then regions_needed -* will indicate the number of file_regions needed in the cache to carry out to -* add the regions for this range. +/* +* Must be called with resv->lock held. +* +* Calling this with regions_needed != NULL will count the number of pages +* to be added but will not modify the linked list. And regions_needed will +* indicate the number of file_regions needed in the cache to carry out to add +* the regions for this range. */ static long add_reservation_in_range (struct resv_map *resv,long f,long t,struct hugetlb_cgroup *h_cg, -struct hstate *h,long *regions_needed, -bool count_only) +struct hstate *h,long *regions_needed) {long add =0 ;struct list_head *head = &resv->regions;
@@ -366,14 +367,14 @@ static long add_reservation_in_range(struct resv_map *resv, long f, long t,*/ if (rg->from > last_accounted_offset) { add += rg->from - last_accounted_offset; -if (!count_only) { +if (!regions_needed) { nrg =get_file_region_entry_from_cache ( resv, last_accounted_offset, rg->from);record_hugetlb_cgroup_uncharge_info (h_cg, h, resv, nrg);list_add (&nrg->link, rg->link.prev);coalesce_file_region (resv, nrg); - }else if (regions_needed) + }else *regions_needed +=1 ; }
@@ -385,13 +386,13 @@ static long add_reservation_in_range(struct resv_map *resv, long f, long t,*/ if (last_accounted_offset < t) { add += t - last_accounted_offset; -if (!count_only) { +if (!regions_needed) { nrg =get_file_region_entry_from_cache ( resv, last_accounted_offset, t);record_hugetlb_cgroup_uncharge_info (h_cg, h, resv, nrg);list_add (&nrg->link, rg->link.prev);coalesce_file_region (resv, nrg); - }else if (regions_needed) + }else *regions_needed +=1 ; }
@@ -484,8 +485,8 @@ static long region_add(struct resv_map *resv, long f, long t, retry:/* Count how many regions are actually needed to execute this add. */ -add_reservation_in_range (resv, f, t,NULL ,NULL , &actual_regions_needed, - true); +add_reservation_in_range (resv, f, t,NULL ,NULL , + &actual_regions_needed);/* * Check for sufficient descriptors in the cache to accommodate
@@ -513,7 +514,7 @@ static long region_add(struct resv_map *resv, long f, long t, goto retry; } - add =add_reservation_in_range (resv, f, t, h_cg, h,NULL , false); + add =add_reservation_in_range (resv, f, t, h_cg, h,NULL ); resv->adds_in_progress -= in_regions_needed;
@@ -549,9 +550,9 @@ static long region_chg(struct resv_map *resv, long f, long t,spin_lock (&resv->lock); -/* Count how many hugepages in this range are NOT respresented. */ +/* Count how many hugepages in this range are NOT represented. */ chg =add_reservation_in_range (resv, f, t,NULL ,NULL , - out_regions_needed, true); + out_regions_needed);if (*out_regions_needed ==0 ) *out_regions_needed =1 ;
Signature · 1 line
2.20.1 (Apple Git-117)
PATCH v4 5/7 mm/hugetlb: a page from buddy is not on any list
In reply to Wei YangThe page allocated from buddy is not on any list, so just use list_add() is enough.
--- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/hugetlb.c +1 -1
@@ -2405,7 +2405,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, h->resv_huge_pages--; }spin_lock (&hugetlb_lock); -list_move (&page->lru, &h->hugepage_activelist); +list_add (&page->lru, &h->hugepage_activelist);/* Fall through */ }hugetlb_cgroup_commit_charge (idx,pages_per_huge_page (h), h_cg, page);
Signature · 1 line
2.20.1 (Apple Git-117)
Re: [Patch v4 5/7] mm/hugetlb: a page from buddy is not on any list
Quoted 15 lines · Wei Yang
The page allocated from buddy is not on any list, so just use
list_add()is enough.Signed-off-by Wei Yang <[email protected]>Reviewed-by Baoquan He <[email protected]>Reviewed-by Mike Kravetz <[email protected]>--- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)mm/hugetlb.c +0
-0
mm/hugetlb.c struct page *alloc_huge_page(struct vm_area_struct *vma, · line 2408
Quoted 5 lines
mm/hugetlb.c +1
-1@@ -2405,7 +2405,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, h->resv_huge_pages--; }spin_lock (&hugetlb_lock); -list_move (&page->lru, &h->hugepage_activelist); +list_add (&page->lru, &h->hugepage_activelist);
Hmm, how does that list_move() actually not crash today?
Page has been taken from free lists, thus there was list_del() and page->lru should be poisoned.list_move() does __list_del_entry() which will either detect the poison with CONFIG_DEBUG_LIST, or crash accessing the poison, no?
Am I missing something or does it mean this code is actually never executed in wild?
mm/hugetlb.c struct page *alloc_huge_page(struct vm_area_struct *vma, · line 2409
Quoted 1 line
mm/hugetlb.c +0
-0/* Fall through */
Maybe delete this comment? This is not a switch statement.
Quoted 3 lines
mm/hugetlb.c +0
-0}hugetlb_cgroup_commit_charge (idx,pages_per_huge_page (h), h_cg, page);
Re: [Patch v4 5/7] mm/hugetlb: a page from buddy is not on any list
Quoted 20 lines · Vlastimil Babka
Quoted 19 lines · Wei Yang
The page allocated from buddy is not on any list, so just use
list_add()is enough.Signed-off-by Wei Yang <[email protected]>Reviewed-by Baoquan He <[email protected]>Reviewed-by Mike Kravetz <[email protected]>--- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)mm/hugetlb.c +0
-1@@ -2405,7 +2405,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, h->resv_huge_pages--; }spin_lock (&hugetlb_lock); -list_move (&page->lru, &h->hugepage_activelist);
Quoted 9 lines
mm/hugetlb.c struct page *alloc_huge_page(struct vm_area_struct *vma, · line 2408
Quoted 1 line
mm/hugetlb.c +1
-0+list_add (&page->lru, &h->hugepage_activelist);Hmm, how does that
list_move()actually not crash today?
Page has been taken from free lists, thus there waslist_del()andpage->lrushould be poisoned.list_move()does__list_del_entry()which will either detect the poison withCONFIG_DEBUG_LIST, or crash accessing the poison, no?
Am I missing something or does it mean this code is actually never executed in wild?
There is not enough context in the diff, but the hugetlb page was not taken from the free list. Rather, it was just created by a call to alloc_buddy_huge_page_with_mpol(). As part of the allocation/creation prep_new_huge_page will be called which will INIT_LIST_HEAD(&page->lru).
Signature · 1 line
Mike Kravetz
Re: [Patch v4 5/7] mm/hugetlb: a page from buddy is not on any list
Quoted 31 lines · Mike Kravetz
Quoted 20 lines · Vlastimil Babka
Quoted 19 lines · Wei Yang
The page allocated from buddy is not on any list, so just use
list_add()is enough.Signed-off-by Wei Yang <[email protected]>Reviewed-by Baoquan He <[email protected]>Reviewed-by Mike Kravetz <[email protected]>--- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)mm/hugetlb.c +0
-1@@ -2405,7 +2405,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, h->resv_huge_pages--; }spin_lock (&hugetlb_lock); -list_move (&page->lru, &h->hugepage_activelist);Quoted 9 lines
mm/hugetlb.c struct page *alloc_huge_page(struct vm_area_struct *vma, · line 2408
Quoted 1 line
mm/hugetlb.c +1
-0+list_add (&page->lru, &h->hugepage_activelist);Hmm, how does that
list_move()actually not crash today?
Page has been taken from free lists, thus there waslist_del()andpage->lrushould be poisoned.list_move()does__list_del_entry()which will either detect the poison withCONFIG_DEBUG_LIST, or crash accessing the poison, no?
Am I missing something or does it mean this code is actually never executed in wild?
Quoted 4 lines
There is not enough context in the diff, but the hugetlb page was not taken from the free list. Rather, it was just created by a call to
alloc_buddy_huge_page_with_mpol(). As part of the allocation/creationprep_new_huge_pagewill be called which willINIT_LIST_HEAD(&page->lru).
Ah so indeed I was missing something :) Thanks. Then this is indeed a an optimization and not a bugfix and doesn't need stable@. Sorry for the noise.
PATCH v4 6/7 mm/hugetlb: narrow the hugetlb_lock protection area during preparing huge page
In reply to Wei Yangset_hugetlb_cgroup_[rsvd] just manipulate page local data, which is not necessary to be protected by hugetlb_lock.
Let's take this out.
--- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/hugetlb.c +1 -1
@@ -1493,9 +1493,9 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) {INIT_LIST_HEAD (&page->lru);set_compound_page_dtor (page,HUGETLB_PAGE_DTOR ); -spin_lock (&hugetlb_lock);set_hugetlb_cgroup (page,NULL );set_hugetlb_cgroup_rsvd (page,NULL ); +spin_lock (&hugetlb_lock); h->nr_huge_pages++; h->nr_huge_pages_node[nid]++;spin_unlock (&hugetlb_lock);
Signature · 1 line
2.20.1 (Apple Git-117)
PATCH v4 7/7 mm/hugetlb: take the free hpage during the iteration directly
In reply to Wei YangFunction dequeue_huge_page_node_exact() iterates the free list and return the first valid free hpage.
Instead of break and check the loop variant, we could return in the loop directly. This could reduce some redundant check.
[[email protected]: points out a logic error]
--- mm/hugetlb.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-)
mm/hugetlb.c +9 -13
@@ -1040,21 +1040,17 @@ static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)if (nocma &&is_migrate_cma_page (page))continue ; -if (!PageHWPoison (page)) -break ; +if (PageHWPoison (page)) +continue ; + +list_move (&page->lru, &h->hugepage_activelist); +set_page_refcounted (page); + h->free_huge_pages--; + h->free_huge_pages_node[nid]--; +return page; } -/* -* if 'non-isolated free hugepage' not found on the list, -* the allocation fails. -*/ -if (&h->hugepage_freelists[nid] == &page->lru) -return NULL ; -list_move (&page->lru, &h->hugepage_activelist); -set_page_refcounted (page); - h->free_huge_pages--; - h->free_huge_pages_node[nid]--; -return page; +return NULL ; } static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
Signature · 1 line
2.20.1 (Apple Git-117)
Re: [Patch v4 7/7] mm/hugetlb: take the free hpage during the iteration directly
Quoted 7 lines · Wei Yang
Function
dequeue_huge_page_node_exact()iterates the free list and return the first valid free hpage.Instead of break and check the loop variant, we could return in the loop directly. This could reduce some redundant check.
Signed-off-by Wei Yang <[email protected]>
Thank you!
Signature · 1 line
Mike Kravetz