Skip to contentSkip to conversation

mm: prefix page_cpupid_reset_last as non-atomic

PATCH In review Nicholas Piggin 2 participants 4 messages Dec 20, 2020 – Dec 21, 2020

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 mm: prefix page_cpupid_reset_last as non-atomic

Add a __ prefix to indicate that it should be used in non-atomic situations (where the page is not subject to concurrent flags access, following the pattern of the other page flags modifications).

This prepares for an atomic version.

---
 include/linux/mm.h | 6 +++---
 mm/page_alloc.c    | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)
include/linux/mm.h +3 -3
index 855161080f18..5b3fc2c8fd8b 100644
@@ -1358,7 +1358,7 @@ static inline int page_cpupid_last(struct page *page)
 {
 	return page->_last_cpupid;
 }
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
@@ -1370,7 +1370,7 @@ static inline int page_cpupid_last(struct page *page)
 
 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
 
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
 }
@@ -1411,7 +1411,7 @@ static inline bool cpupid_pid_unset(int cpupid)
 	return true;
 }
 
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 }
 
mm/page_alloc.c +2 -2
index 3beeb8d722f3..ead28946f3ce 100644
@@ -1260,7 +1260,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	if (bad)
 		return false;
 
-	page_cpupid_reset_last(page);
+	__page_cpupid_reset_last(page);
 	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
 	reset_page_owner(page, order);
 
@@ -1453,7 +1453,7 @@ static void __meminit __init_single_page(struct page *page, unsigned long pfn,
 	set_page_links(page, zone, nid, pfn);
 	init_page_count(page);
 	page_mapcount_reset(page);
-	page_cpupid_reset_last(page);
+	__page_cpupid_reset_last(page);
 	page_kasan_tag_reset(page);
 
 	INIT_LIST_HEAD(&page->lru);
Signature · 1 line
2.23.0
#1 lore raw

PATCH 2/3 mm: add page_cpupid_reset_last atomic variant

This can be used when there is concurrency on the struct page. If cpupid is not in page flags, it can use a store.

Use this to replace the open-coded reset in wp_page_reuse.

---
 include/linux/mm.h | 15 +++++++++++++++
 mm/memory.c        |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)
include/linux/mm.h +15 -0
index 5b3fc2c8fd8b..6ece500464e0 100644
@@ -1362,6 +1362,11 @@ static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	WRITE_ONCE(page->_last_cpupid, -1 & LAST_CPUPID_MASK);
+}
 #else
 static inline int page_cpupid_last(struct page *page)
 {
@@ -1374,6 +1379,12 @@ static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+}
+
 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
 #else /* !CONFIG_NUMA_BALANCING */
 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
@@ -1415,6 +1426,10 @@ static inline void __page_cpupid_reset_last(struct page *page)
 {
 }
 
+static inline void page_cpupid_reset_last(struct page *page)
+{
+}
+
 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
 {
 	return false;
mm/memory.c +1 -1
index 7d608765932b..990e5d704c08 100644
@@ -2798,7 +2798,7 @@ static inline void wp_page_reuse(struct vm_fault *vmf)
 	 * unrelated process.
 	 */
 	if (page)
-		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+		page_cpupid_reset_last(page);
 
 	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
 	entry = pte_mkyoung(vmf->orig_pte);
Signature · 1 line
2.23.0
#2 lore raw

PATCH 3/3 mm: skip page_cpupid_reset_last atomic update if the value is already reset

In reply to Nicholas Piggin

Improves performance of a PROT_READ -> PROT_READ|PROT_WRITE -> store microbenchmark (which goes via the wp_page_reuse fault path) by about 10% on a POWER9.

---
 include/linux/mm.h | 3 +++
 1 file changed, 3 insertions(+)
include/linux/mm.h +3 -0
index 6ece500464e0..c0a9e8de0738 100644
@@ -1382,6 +1382,9 @@ static inline void __page_cpupid_reset_last(struct page *page)
 
 static inline void page_cpupid_reset_last(struct page *page)
 {
+	if (page_cpupid_last(page) == (-1 & LAST_CPUPID_MASK))
+		return;
+
 	page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
 }
 
Signature · 1 line
2.23.0
#3 lore raw
In reply to Nicholas Piggin

Hi, Nicholas,

Quoted 18 lines · Nicholas Piggin

Add a __ prefix to indicate that it should be used in non-atomic situations (where the page is not subject to concurrent flags access, following the pattern of the other page flags modifications).

This prepares for an atomic version.

---
 include/linux/mm.h | 6 +++---
 mm/page_alloc.c    | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)
include/linux/mm.h +0 -0
index 855161080f18..5b3fc2c8fd8b 100644

include/linux/mm.h static inline int page_cpupid_last(struct page *page) · line 1363

Quoted 7 lines
include/linux/mm.h +1 -1
@@ -1358,7 +1358,7 @@ static inline int page_cpupid_last(struct page *page)
 {
        return page->_last_cpupid;
 }
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
        page->_last_cpupid = -1 & LAST_CPUPID_MASK;

How about replace all these

-1 & LAST_CPUPID_MASK

and

(1 << LAST_CPUPID_SHIFT) - 1)

with

LAST_CPUPID_MASK

I think that can improve the code readability a little.

Best Regards,
Huang, Ying